diff --git a/clang-tools-extra/clang-query/Query.cpp b/clang-tools-extra/clang-query/Query.cpp
index ca2a285e9eb7..e33612a2e16d 100644
--- a/clang-tools-extra/clang-query/Query.cpp
+++ b/clang-tools-extra/clang-query/Query.cpp
@@ -156,8 +156,7 @@ bool MatchQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
         if (QS.DetailedASTOutput) {
           OS << "Binding for \"" << BI->first << "\":\n";
           const ASTContext &Ctx = AST->getASTContext();
-          const SourceManager &SM = Ctx.getSourceManager();
-          ASTDumper Dumper(OS, Ctx, SM.getDiagnostics().getShowColors());
+          ASTDumper Dumper(OS, Ctx, AST->getDiagnostics().getShowColors());
           Dumper.SetTraversalKind(QS.TK);
           Dumper.Visit(BI->second);
           OS << "\n";
diff --git a/clang-tools-extra/clang-query/tool/ClangQuery.cpp b/clang-tools-extra/clang-query/tool/ClangQuery.cpp
index 31c7f12251c9..45a355073945 100644
--- a/clang-tools-extra/clang-query/tool/ClangQuery.cpp
+++ b/clang-tools-extra/clang-query/tool/ClangQuery.cpp
@@ -49,6 +49,14 @@ using namespace llvm;
 static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
 static cl::OptionCategory ClangQueryCategory("clang-query options");
 
+static cl::opt<bool>
+    UseColor("use-color",
+             cl::desc(
+                 R"(Use colors in detailed AST output. If not set, colors
+will be used if the terminal connected to
+standard output supports colors.)"),
+             cl::init(false), cl::cat(ClangQueryCategory));
+
 static cl::list<std::string> Commands("c", cl::desc("Specify command to run"),
                                       cl::value_desc("command"),
                                       cl::cat(ClangQueryCategory));
@@ -109,6 +117,19 @@ int main(int argc, const char **argv) {
 
   ClangTool Tool(OptionsParser->getCompilations(),
                  OptionsParser->getSourcePathList());
+
+  if (UseColor.getNumOccurrences() > 0) {
+    ArgumentsAdjuster colorAdjustor = [](const CommandLineArguments &Args, StringRef /*unused*/) {
+      CommandLineArguments AdjustedArgs = Args;
+      if (UseColor)
+        AdjustedArgs.push_back("-fdiagnostics-color");
+      else
+        AdjustedArgs.push_back("-fno-diagnostics-color");
+      return AdjustedArgs;
+    };
+    Tool.appendArgumentsAdjuster(colorAdjustor);
+  }
+
   std::vector<std::unique_ptr<ASTUnit>> ASTs;
   int ASTStatus = 0;
   switch (Tool.buildASTs(ASTs)) {
diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
index 79e94ef6fe37..675e721c7e64 100644
--- a/clang-tools-extra/clangd/Config.h
+++ b/clang-tools-extra/clangd/Config.h
@@ -28,6 +28,7 @@
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
 #include <string>
 #include <vector>
 
@@ -77,6 +78,12 @@ struct Config {
     llvm::Optional<ExternalIndexSpec> External;
   } Index;
 
+  /// Controls warnings and errors when parsing code.
+  struct {
+    bool SuppressAll = false;
+    llvm::StringSet<> Suppress;
+  } Diagnostics;
+
   /// Style of the codebase.
   struct {
     // Namespaces that should always be fully qualified, meaning no "using"
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index 2040ea4649fe..f2e6d544e6c9 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -27,6 +27,7 @@
 #include "Config.h"
 #include "ConfigFragment.h"
 #include "ConfigProvider.h"
+#include "Diagnostics.h"
 #include "Features.inc"
 #include "TidyProvider.h"
 #include "support/Logger.h"
@@ -187,6 +188,7 @@ struct FragmentCompiler {
     compile(std::move(F.If));
     compile(std::move(F.CompileFlags));
     compile(std::move(F.Index));
+    compile(std::move(F.Diagnostics));
     compile(std::move(F.ClangTidy));
   }
 
@@ -328,6 +330,27 @@ struct FragmentCompiler {
     });
   }
 
+  void compile(Fragment::DiagnosticsBlock &&F) {
+    std::vector<llvm::StringRef> Normalized;
+    for (const auto &Suppressed : F.Suppress) {
+      if (*Suppressed == "*") {
+        Out.Apply.push_back([&](const Params &, Config &C) {
+          C.Diagnostics.SuppressAll = true;
+          C.Diagnostics.Suppress.clear();
+        });
+        return;
+      }
+      Normalized.push_back(normalizeSuppressedCode(*Suppressed));
+    }
+    if (!Normalized.empty())
+      Out.Apply.push_back([Normalized](const Params &, Config &C) {
+        if (C.Diagnostics.SuppressAll)
+          return;
+        for (llvm::StringRef N : Normalized)
+          C.Diagnostics.Suppress.insert(N);
+      });
+  }
+
   void compile(Fragment::StyleBlock &&F) {
     if (!F.FullyQualifiedNamespaces.empty()) {
       std::vector<std::string> FullyQualifiedNamespaces;
diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
index 0e4ce638fc72..c491ec5ee68c 100644
--- a/clang-tools-extra/clangd/ConfigFragment.h
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -181,6 +181,24 @@ struct Fragment {
   };
   IndexBlock Index;
 
+  /// Controls behavior of diagnostics (errors and warnings).
+  struct DiagnosticsBlock {
+    /// Diagnostic codes that should be suppressed.
+    ///
+    /// Valid values are:
+    /// - *, to disable all diagnostics
+    /// - diagnostic codes exposed by clangd (e.g unknown_type, -Wunused-result)
+    /// - clang internal diagnostic codes (e.g. err_unknown_type)
+    /// - warning categories (e.g. unused-result)
+    /// - clang-tidy check names (e.g. bugprone-narrowing-conversions)
+    ///
+    /// This is a simple filter. Diagnostics can be controlled in other ways
+    /// (e.g. by disabling a clang-tidy check, or the -Wunused compile flag).
+    /// This often has other advantages, such as skipping some analysis.
+    std::vector<Located<std::string>> Suppress;
+  };
+  DiagnosticsBlock Diagnostics;
+
   // Describes the style of the codebase, beyond formatting.
   struct StyleBlock {
     // Namespaces that should always be fully qualified, meaning no "using"
@@ -195,6 +213,7 @@ struct Fragment {
   ///
   /// The settings are merged with any settings found in .clang-tidy
   /// configiration files with these ones taking precedence.
+  // FIXME: move this to Diagnostics.Tidy.
   struct ClangTidyBlock {
     std::vector<Located<std::string>> Add;
     /// List of checks to disable.
diff --git a/clang-tools-extra/clangd/Diagnostics.cpp b/clang-tools-extra/clangd/Diagnostics.cpp
index 7b59cedb0f67..d2982636c807 100644
--- a/clang-tools-extra/clangd/Diagnostics.cpp
+++ b/clang-tools-extra/clangd/Diagnostics.cpp
@@ -801,5 +801,23 @@ void StoreDiags::flushLastDiag() {
   Output.push_back(std::move(*LastDiag));
 }
 
+bool isBuiltinDiagnosticSuppressed(unsigned ID,
+                                   const llvm::StringSet<> &Suppress) {
+  if (const char *CodePtr = getDiagnosticCode(ID)) {
+    if (Suppress.contains(normalizeSuppressedCode(CodePtr)))
+      return true;
+  }
+  StringRef Warning = DiagnosticIDs::getWarningOptionForDiag(ID);
+  if (!Warning.empty() && Suppress.contains(Warning))
+    return true;
+  return false;
+}
+
+llvm::StringRef normalizeSuppressedCode(llvm::StringRef Code) {
+  Code.consume_front("err_");
+  Code.consume_front("-W");
+  return Code;
+}
+
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/Diagnostics.h b/clang-tools-extra/clangd/Diagnostics.h
index e1c3bcafcbf7..6a432db23cc2 100644
--- a/clang-tools-extra/clangd/Diagnostics.h
+++ b/clang-tools-extra/clangd/Diagnostics.h
@@ -159,6 +159,15 @@ class StoreDiags : public DiagnosticConsumer {
   bool LastPrimaryDiagnosticWasSuppressed = false;
 };
 
+/// Determine whether a (non-clang-tidy) diagnostic is suppressed by config.
+bool isBuiltinDiagnosticSuppressed(unsigned ID,
+                                   const llvm::StringSet<> &Suppressed);
+/// Take a user-specified diagnostic code, and convert it to a normalized form
+/// stored in the config and consumed by isBuiltinDiagnosticsSuppressed.
+///
+/// (This strips err_ and -W prefix so we can match with or without them.)
+llvm::StringRef normalizeSuppressedCode(llvm::StringRef);
+
 } // namespace clangd
 } // namespace clang
 
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index a8c4eea54540..403d3fe3e64f 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -12,6 +12,7 @@
 #include "../clang-tidy/ClangTidyModuleRegistry.h"
 #include "AST.h"
 #include "Compiler.h"
+#include "Config.h"
 #include "Diagnostics.h"
 #include "Headers.h"
 #include "IncludeFixer.h"
@@ -315,12 +316,18 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
       Check->registerMatchers(&CTFinder);
     }
 
-    if (!CTChecks.empty()) {
-      ASTDiags.setLevelAdjuster([&CTContext](DiagnosticsEngine::Level DiagLevel,
-                                             const clang::Diagnostic &Info) {
+    ASTDiags.setLevelAdjuster([&, &Cfg(Config::current())](
+                                  DiagnosticsEngine::Level DiagLevel,
+                                  const clang::Diagnostic &Info) {
+      if (Cfg.Diagnostics.SuppressAll ||
+          isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress))
+        return DiagnosticsEngine::Ignored;
+      if (!CTChecks.empty()) {
         std::string CheckName = CTContext->getCheckName(Info.getID());
         bool IsClangTidyDiag = !CheckName.empty();
         if (IsClangTidyDiag) {
+          if (Cfg.Diagnostics.Suppress.contains(CheckName))
+            return DiagnosticsEngine::Ignored;
           // Check for suppression comment. Skip the check for diagnostics not
           // in the main file, because we don't want that function to query the
           // source buffer for preamble files. For the same reason, we ask
@@ -342,9 +349,9 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
             return DiagnosticsEngine::Error;
           }
         }
-        return DiagLevel;
-      });
-    }
+      }
+      return DiagLevel;
+    });
   }
 
   // Add IncludeFixer which can recover diagnostics caused by missing includes
diff --git a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
index 578f69821e4d..dde8e644f15a 100644
--- a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
@@ -11,6 +11,7 @@
 #include "ConfigTesting.h"
 #include "Features.inc"
 #include "TestFS.h"
+#include "clang/Basic/DiagnosticSema.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
@@ -30,6 +31,7 @@ using ::testing::ElementsAre;
 using ::testing::IsEmpty;
 using ::testing::SizeIs;
 using ::testing::StartsWith;
+using ::testing::UnorderedElementsAre;
 
 class ConfigCompileTests : public ::testing::Test {
 protected:
@@ -183,6 +185,39 @@ TEST_F(ConfigCompileTests, PathSpecMatch) {
   }
 }
 
+TEST_F(ConfigCompileTests, DiagnosticSuppression) {
+  Frag.Diagnostics.Suppress.emplace_back("bugprone-use-after-move");
+  Frag.Diagnostics.Suppress.emplace_back("unreachable-code");
+  Frag.Diagnostics.Suppress.emplace_back("-Wunused-variable");
+  Frag.Diagnostics.Suppress.emplace_back("typecheck_bool_condition");
+  Frag.Diagnostics.Suppress.emplace_back("err_unexpected_friend");
+  Frag.Diagnostics.Suppress.emplace_back("warn_alloca");
+  EXPECT_TRUE(compileAndApply());
+  EXPECT_THAT(Conf.Diagnostics.Suppress.keys(),
+              UnorderedElementsAre("bugprone-use-after-move",
+                                   "unreachable-code", "unused-variable",
+                                   "typecheck_bool_condition",
+                                   "unexpected_friend", "warn_alloca"));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::warn_unreachable,
+                                            Conf.Diagnostics.Suppress));
+  // Subcategory not respected/suppressed.
+  EXPECT_FALSE(isBuiltinDiagnosticSuppressed(diag::warn_unreachable_break,
+                                             Conf.Diagnostics.Suppress));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::warn_unused_variable,
+                                            Conf.Diagnostics.Suppress));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::err_typecheck_bool_condition,
+                                            Conf.Diagnostics.Suppress));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::err_unexpected_friend,
+                                            Conf.Diagnostics.Suppress));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::warn_alloca,
+                                            Conf.Diagnostics.Suppress));
+
+  Frag.Diagnostics.Suppress.emplace_back("*");
+  EXPECT_TRUE(compileAndApply());
+  EXPECT_TRUE(Conf.Diagnostics.SuppressAll);
+  EXPECT_THAT(Conf.Diagnostics.Suppress, IsEmpty());
+}
+
 TEST_F(ConfigCompileTests, Tidy) {
   Frag.ClangTidy.Add.emplace_back("bugprone-use-after-move");
   Frag.ClangTidy.Add.emplace_back("llvm-*");
diff --git a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
index 4039a9be71a0..25d468ba604a 100644
--- a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
@@ -98,7 +98,7 @@ TEST(ParseYAML, Locations) {
             YAML.range());
 }
 
-TEST(ParseYAML, Diagnostics) {
+TEST(ParseYAML, ConfigDiagnostics) {
   CapturedDiags Diags;
   Annotations YAML(R"yaml(
 If:
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index c6a14aeeb469..b1e748d18097 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Annotations.h"
+#include "Config.h"
 #include "Diagnostics.h"
 #include "ParsedAST.h"
 #include "Protocol.h"
@@ -16,6 +17,7 @@
 #include "TestTU.h"
 #include "TidyProvider.h"
 #include "index/MemIndex.h"
+#include "support/Context.h"
 #include "support/Path.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticSema.h"
@@ -371,6 +373,28 @@ TEST(DiagnosticTest, NoMultipleDiagnosticInFlight) {
           DiagSource(Diag::ClangTidy), DiagName("modernize-loop-convert"))));
 }
 
+TEST(DiagnosticTest, RespectsDiagnosticConfig) {
+  Annotations Main(R"cpp(
+    // error-ok
+    void x() {
+      [[unknown]]();
+      $ret[[return]] 42;
+    }
+  )cpp");
+  auto TU = TestTU::withCode(Main.code());
+  EXPECT_THAT(
+      TU.build().getDiagnostics(),
+      ElementsAre(Diag(Main.range(), "use of undeclared identifier 'unknown'"),
+                  Diag(Main.range("ret"),
+                       "void function 'x' should not return a value")));
+  Config Cfg;
+  Cfg.Diagnostics.Suppress.insert("return-type");
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+  EXPECT_THAT(TU.build().getDiagnostics(),
+              ElementsAre(Diag(Main.range(),
+                               "use of undeclared identifier 'unknown'")));
+}
+
 TEST(DiagnosticTest, ClangTidySuppressionComment) {
   Annotations Main(R"cpp(
     int main() {
diff --git a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
index 7e36cff7afa6..924cfd03cba7 100644
--- a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
@@ -1838,6 +1838,20 @@ TEST_F(SymbolCollectorTest, UndefOfModuleMacro) {
   EXPECT_THAT(TU.headerSymbols(), Not(Contains(QName("X"))));
 }
 
+TEST_F(SymbolCollectorTest, NoCrashOnObjCMethodCStyleParam) {
+  auto TU = TestTU::withCode(R"objc(
+    @interface Foo
+    - (void)fun:(bool)foo, bool bar;
+    @end
+  )objc");
+  TU.ExtraArgs.push_back("-xobjective-c++");
+
+  TU.build();
+  // We mostly care about not crashing.
+  EXPECT_THAT(TU.headerSymbols(),
+              UnorderedElementsAre(QName("Foo"), QName("Foo::fun:")));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 27dcee83a538..3141dd5510fc 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -195,23 +195,84 @@ the configuration (without a prefix: ``Auto``).
 
 
 
-**AlignConsecutiveAssignments** (``bool``)
-  If ``true``, aligns consecutive assignments.
+**AlignConsecutiveAssignments** (``AlignConsecutiveStyle``)
+  Style of aligning consecutive assignments.
 
-  This will align the assignment operators of consecutive lines. This
-  will result in formattings like
+  ``Consecutive`` will result in formattings like:
 
   .. code-block:: c++
 
-    int aaaa = 12;
-    int b    = 23;
-    int ccc  = 23;
+    int a            = 1;
+    int somelongname = 2;
+    double c         = 3;
 
-**AlignConsecutiveBitFields** (``bool``)
-  If ``true``, aligns consecutive bitfield members.
+  Possible values:
+
+  * ``ACS_None`` (in configuration: ``None``)
+     Do not align assignments on consecutive lines.
+
+  * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+     Align assignments on consecutive lines. This will result in
+     formattings like:
+
+     .. code-block:: c++
+
+       int a            = 1;
+       int somelongname = 2;
+       double c         = 3;
+
+       int d = 3;
+       /* A comment. */
+       double e = 4;
+
+  * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+     Same as ACS_Consecutive, but also spans over empty lines, e.g.
+
+     .. code-block:: c++
+
+       int a            = 1;
+       int somelongname = 2;
+       double c         = 3;
+
+       int d            = 3;
+       /* A comment. */
+       double e = 4;
+
+  * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments, e.g.
+
+     .. code-block:: c++
+
+       int a            = 1;
+       int somelongname = 2;
+       double c         = 3;
+
+       int d    = 3;
+       /* A comment. */
+       double e = 4;
+
+  * ``ACS_AcrossEmptyLinesAndComments``
+    (in configuration: ``AcrossEmptyLinesAndComments``)
+
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments and empty lines, e.g.
+
+     .. code-block:: c++
+
+       int a            = 1;
+       int somelongname = 2;
+       double c         = 3;
+
+       int d            = 3;
+       /* A comment. */
+       double e         = 4;
+
+**AlignConsecutiveBitFields** (``AlignConsecutiveStyle``)
+  Style of aligning consecutive bit field.
 
-  This will align the bitfield separators of consecutive lines. This
-  will result in formattings like
+  ``Consecutive`` will align the bitfield separators of consecutive lines.
+  This will result in formattings like:
 
   .. code-block:: c++
 
@@ -219,23 +280,146 @@ the configuration (without a prefix: ``Auto``).
     int b    : 12;
     int ccc  : 8;
 
-**AlignConsecutiveDeclarations** (``bool``)
-  If ``true``, aligns consecutive declarations.
+  Possible values:
+
+  * ``ACS_None`` (in configuration: ``None``)
+     Do not align bit fields on consecutive lines.
+
+  * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+     Align bit fields on consecutive lines. This will result in
+     formattings like:
+
+     .. code-block:: c++
+
+       int aaaa : 1;
+       int b    : 12;
+       int ccc  : 8;
+
+       int d : 2;
+       /* A comment. */
+       int ee : 3;
+
+  * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+     Same as ACS_Consecutive, but also spans over empty lines, e.g.
+
+     .. code-block:: c++
+
+       int aaaa : 1;
+       int b    : 12;
+       int ccc  : 8;
+
+       int d    : 2;
+       /* A comment. */
+       int ee : 3;
+
+  * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments, e.g.
+
+     .. code-block:: c++
+
+       int aaaa : 1;
+       int b    : 12;
+       int ccc  : 8;
+
+       int d  : 2;
+       /* A comment. */
+       int ee : 3;
+
+  * ``ACS_AcrossEmptyLinesAndComments``
+    (in configuration: ``AcrossEmptyLinesAndComments``)
+
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments and empty lines, e.g.
+
+     .. code-block:: c++
+
+       int aaaa : 1;
+       int b    : 12;
+       int ccc  : 8;
+
+       int d    : 2;
+       /* A comment. */
+       int ee   : 3;
+
+**AlignConsecutiveDeclarations** (``AlignConsecutiveStyle``)
+  Style of aligning consecutive declarations.
 
-  This will align the declaration names of consecutive lines. This
-  will result in formattings like
+  ``Consecutive`` will align the declaration names of consecutive lines.
+  This will result in formattings like:
 
   .. code-block:: c++
 
     int         aaaa = 12;
     float       b = 23;
-    std::string ccc = 23;
+    std::string ccc;
 
-**AlignConsecutiveMacros** (``bool``)
-  If ``true``, aligns consecutive C/C++ preprocessor macros.
+  Possible values:
+
+  * ``ACS_None`` (in configuration: ``None``)
+     Do not align bit declarations on consecutive lines.
+
+  * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+     Align declarations on consecutive lines. This will result in
+     formattings like:
+
+     .. code-block:: c++
+
+       int         aaaa = 12;
+       float       b = 23;
+       std::string ccc;
+
+       int a = 42;
+       /* A comment. */
+       bool c = false;
+
+  * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+     Same as ACS_Consecutive, but also spans over empty lines, e.g.
+
+     .. code-block:: c++
+
+       int         aaaa = 12;
+       float       b = 23;
+       std::string ccc;
+
+       int         a = 42;
+       /* A comment. */
+       bool c = false;
+
+  * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments, e.g.
+
+     .. code-block:: c++
+
+       int         aaaa = 12;
+       float       b = 23;
+       std::string ccc;
+
+       int  a = 42;
+       /* A comment. */
+       bool c = false;
+
+  * ``ACS_AcrossEmptyLinesAndComments``
+    (in configuration: ``AcrossEmptyLinesAndComments``)
+
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments and empty lines, e.g.
+
+     .. code-block:: c++
+
+       int         aaaa = 12;
+       float       b = 23;
+       std::string ccc;
+
+       int         a = 42;
+       /* A comment. */
+       bool        c = false;
+
+**AlignConsecutiveMacros** (``AlignConsecutiveStyle``)
+  Style of aligning consecutive macro definitions.
 
-  This will align C/C++ preprocessor macros of consecutive lines.
-  Will result in formattings like
+  ``Consecutive`` will result in formattings like:
 
   .. code-block:: c++
 
@@ -245,6 +429,68 @@ the configuration (without a prefix: ``Auto``).
     #define foo(x)           (x * x)
     #define bar(y, z)        (y + z)
 
+  Possible values:
+
+  * ``ACS_None`` (in configuration: ``None``)
+     Do not align macro definitions on consecutive lines.
+
+  * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+     Align macro definitions on consecutive lines. This will result in
+     formattings like:
+
+     .. code-block:: c++
+
+       #define SHORT_NAME       42
+       #define LONGER_NAME      0x007f
+       #define EVEN_LONGER_NAME (2)
+
+       #define foo(x) (x * x)
+       /* some comment */
+       #define bar(y, z) (y + z)
+
+  * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+     Same as ACS_Consecutive, but also spans over empty lines, e.g.
+
+     .. code-block:: c++
+
+       #define SHORT_NAME       42
+       #define LONGER_NAME      0x007f
+       #define EVEN_LONGER_NAME (2)
+
+       #define foo(x)           (x * x)
+       /* some comment */
+       #define bar(y, z) (y + z)
+
+  * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments, e.g.
+
+     .. code-block:: c++
+
+       #define SHORT_NAME       42
+       #define LONGER_NAME      0x007f
+       #define EVEN_LONGER_NAME (2)
+
+       #define foo(x)    (x * x)
+       /* some comment */
+       #define bar(y, z) (y + z)
+
+  * ``ACS_AcrossEmptyLinesAndComments``
+    (in configuration: ``AcrossEmptyLinesAndComments``)
+
+     Same as ACS_Consecutive, but also spans over lines only containing
+     comments and empty lines, e.g.
+
+     .. code-block:: c++
+
+       #define SHORT_NAME       42
+       #define LONGER_NAME      0x007f
+       #define EVEN_LONGER_NAME (2)
+
+       #define foo(x)           (x * x)
+       /* some comment */
+       #define bar(y, z)        (y + z)
+
 **AlignEscapedNewlines** (``EscapedNewlineAlignmentStyle``)
   Options for aligning backslashes in escaped newlines.
 
diff --git a/clang/docs/OpenCLSupport.rst b/clang/docs/OpenCLSupport.rst
index 5be7e91adcaf..0eaf0f300ee4 100644
--- a/clang/docs/OpenCLSupport.rst
+++ b/clang/docs/OpenCLSupport.rst
@@ -112,6 +112,28 @@ Feel free to contact us on `cfe-dev
 <https://lists.llvm.org/mailman/listinfo/cfe-dev>`_ or via `Bugzilla
 <https://bugs.llvm.org/>`__.
 
+Fast builtin function declarations
+----------------------------------
+
+In addition to regular header includes with builtin types and functions using
+``-finclude-default-header`` explained in :doc:`UsersManual`, clang
+supports a fast mechanism to declare builtin functions with
+``-fdeclare-opencl-builtins``. This does not declare the builtin types and
+therefore it has to be used in combination with ``-finclude-default-header``
+if full functionality is required.
+
+**Example of Use**:
+
+    .. code-block:: console
+ 
+      $ clang -Xclang -finclude-default-header test.cl
+
+Note that this is a frontend-only flag and therefore it requires the use of
+flags that forward options to the frontend, e.g. ``-cc1`` or ``-Xclang``.
+
+As this feature is still in experimental phase some changes might still occur
+on the command line interface side.
+
 C++ libraries for OpenCL
 ------------------------
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7fcae5bce164..a595ee2719bd 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -289,9 +289,11 @@ clang-format
 
 - Option ``IndentRequires`` has been added to indent the ``requires`` keyword
   in templates.
+
 - Option ``BreakBeforeConceptDeclarations`` has been added to aid the formatting of concepts.
 
-- Option ``IndentPragmas`` has been added to allow #pragma to indented with the current scope level. This is especially useful when using #pragma to mark OpenMP sections of code.
+- Option ``IndentPragmas`` has been added to allow #pragma to indented with the current scope
+  level. This is especially useful when using #pragma to mark OpenMP sections of code.
 
 - Option ``SpaceBeforeCaseColon`` has been added to add a space before the
   colon in a case or default statement.
@@ -300,6 +302,9 @@ clang-format
   macros which are not parsed as a type in front of a statement. See
   the documentation for an example.
 
+- Options ``AlignConsecutiveAssignments``, ``AlignConsecutiveBitFields``,
+  ``AlignConsecutiveDeclarations`` and ``AlignConsecutiveMacros`` have been modified to allow
+  alignment across empty lines and/or comments.
 
 libclang
 --------
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index a7b698d77c47..fe944a0166b8 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -3021,6 +3021,11 @@ To enable modules for OpenCL:
 
      $ clang -target spir-unknown-unknown -c -emit-llvm -Xclang -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=<path to the generated module> test.cl
 
+Another way to circumvent long parsing latency for the OpenCL builtin
+declarations is to use mechanism enabled by ``-fdeclare-opencl-builtins`` flag
+that is available as an experimental feature (see more information in
+:doc:`OpenCLSupport`).
+
 OpenCL Extensions
 -----------------
 
diff --git a/clang/docs/tools/dump_format_style.py b/clang/docs/tools/dump_format_style.py
index d01c823a9a20..fd5837055789 100755
--- a/clang/docs/tools/dump_format_style.py
+++ b/clang/docs/tools/dump_format_style.py
@@ -42,7 +42,7 @@ def __init__(self, name, type, comment):
   def __str__(self):
     s = '**%s** (``%s``)\n%s' % (self.name, self.type,
                                  doxygen2rst(indent(self.comment, 2)))
-    if self.enum:
+    if self.enum and self.enum.values:
       s += indent('\n\nPossible values:\n\n%s\n' % self.enum, 2)
     if self.nested_struct:
       s += indent('\n\nNested configuration flags:\n\n%s\n' %self.nested_struct,
@@ -104,13 +104,18 @@ def __str__(self):
         doxygen2rst(indent(self.comment, 2)))
 
 def clean_comment_line(line):
-  match = re.match(r'^/// \\code(\{.(\w+)\})?$', line)
+  match = re.match(r'^/// (?P<indent> +)?\\code(\{.(?P<lang>\w+)\})?$', line)
   if match:
-    lang = match.groups()[1]
+    indent = match.group('indent')
+    if not indent:
+      indent = ''
+    lang = match.group('lang')
     if not lang:
       lang = 'c++'
-    return '\n.. code-block:: %s\n\n' % lang
-  if line == '/// \\endcode':
+    return '\n%s.. code-block:: %s\n\n' % (indent, lang)
+
+  endcode_match = re.match(r'^/// +\\endcode$', line)
+  if endcode_match:
     return ''
   return line[4:] + '\n'
 
@@ -184,7 +189,9 @@ class State(object):
         state = State.InStruct
         enums[enum.name] = enum
       else:
-        raise Exception('Invalid format, expected enum field comment or };')
+        # Enum member without documentation. Must be documented where the enum
+        # is used.
+        pass
     elif state == State.InEnumMemberComment:
       if line.startswith('///'):
         comment += clean_comment_line(line)
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 9ae17692a9fa..b13d2b8c3d7c 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -2406,12 +2406,10 @@ class ASTContext : public RefCountedBase<ASTContext> {
         return (*SuperTnullability == NullabilityKind::NonNull &&
                 *SubTnullability == NullabilityKind::Nullable);
       }
-      else {
-        // For the return type, it's okay for the superclass method to specify
-        // "nullable" and the subclass method specify "nonnull"
-        return (*SuperTnullability == NullabilityKind::Nullable &&
-                *SubTnullability == NullabilityKind::NonNull);
-      }
+      // For the return type, it's okay for the superclass method to specify
+      // "nullable" and the subclass method specify "nonnull"
+      return (*SuperTnullability == NullabilityKind::Nullable &&
+              *SubTnullability == NullabilityKind::NonNull);
     }
     return true;
   }
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index 9353be1753b0..801916c3ab94 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -10,19 +10,25 @@
 //
 //===----------------------------------------------------------------------===//
 
-// Macro OPENCLEXT or OPENCLEXT_INTERNAL can be defined to enumerate the
+// Macro OPENCLEXTNAME or OPENCL_GENERIC_EXTENSION can be defined to enumerate all
 // OpenCL extensions listed in this file.
 //
-// If the extensions are to be enumerated without the supported OpenCL version,
-// define OPENCLEXT(ext) where ext is the name of the extension.
-//
-// If the extensions are to be enumerated with supported OpenCL version,
-// define OPENCLEXT_INTERNAL(ext, avail, core) where
+// If extensions are to be enumerated with information about whether
+// an extension is core or optional core and minimum OpenCL version
+// when an extension becomes available,
+// define OPENCL_GENERIC_EXTENSION(ext, avail, core, opt) where
 //   ext - name of the extension or optional core feature.
 //   avail - minimum OpenCL version supporting it.
-//   core - minimum OpenCL version when the extension becomes optional core
-//          feature or core feature. ~0U indicates not a core feature or an
-//          optional core feature.
+//   core - OpenCL versions mask when the extension becomes core feature.
+//          0U indicates not a core feature.
+//   opt - OpenCL versions mask when the extension becomes optional core
+//         feature. 0U indicates not a optional core feature.
+//
+// If extensions are to be enumerated without any information,
+// define OPENCLEXTNAME(ext) where ext is the name of the extension.
+//
+// Difference between optional core feature and core feature is that the 
+// later is unconditionally supported in specific OpenCL version.
 //
 // As per The OpenCL Extension Specification, Section 1.2, in this file, an
 // extension is defined if and only it either:
@@ -32,63 +38,72 @@
 // For such an extension, a preprocessor #define that matches the extension
 // name must be created and a #pragma is required if and only if the
 // compilation flow is impacted, e.g. due to a difference of syntax or
-// semantics in the language compared to the core standard.
+// semantics in the language compared to the core standard. #pragma directive
+// has no effect for optional core and core features.
 
-#ifndef OPENCLEXT_INTERNAL
-#ifndef OPENCLEXT
-#pragma error "macro OPENCLEXT or OPENCLEXT_INTERNAL is required"
+#ifndef OPENCL_GENERIC_EXTENSION
+#ifndef OPENCLEXTNAME
+#pragma error "macro OPENCLEXTNAME or OPENCL_GENERIC_EXTENSION is required"
 #else
-#define OPENCLEXT_INTERNAL(ext, ...) OPENCLEXT(ext)
-#endif // OPENCLEXT
-#endif // OPENCLEXT_INTERNAL
+#define OPENCL_GENERIC_EXTENSION(ext, ...) OPENCLEXTNAME(ext)
+#endif // OPENCLEXTNAME
+#endif // OPENCL_GENERIC_EXTENSION
+
+// Declaration helpers
+#define OPENCL_EXTENSION(ext, avail) OPENCL_GENERIC_EXTENSION(ext, avail, 0U, 0U)
+#define OPENCL_COREFEATURE(ext, avail, core)  OPENCL_GENERIC_EXTENSION(ext, avail, core, 0U)
+#define OPENCL_OPTIONALCOREFEATURE(ext, avail, opt) OPENCL_GENERIC_EXTENSION(ext, avail, 0U, opt)
 
 // OpenCL 1.0.
-OPENCLEXT_INTERNAL(cl_khr_3d_image_writes, 100, 200)
-OPENCLEXT_INTERNAL(cl_khr_byte_addressable_store, 100, 110)
-OPENCLEXT_INTERNAL(cl_khr_fp16, 100, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_fp64, 100, 120)
-OPENCLEXT_INTERNAL(cl_khr_global_int32_base_atomics, 100, 110)
-OPENCLEXT_INTERNAL(cl_khr_global_int32_extended_atomics, 100, 110)
-OPENCLEXT_INTERNAL(cl_khr_local_int32_base_atomics, 100, 110)
-OPENCLEXT_INTERNAL(cl_khr_local_int32_extended_atomics, 100, 110)
-OPENCLEXT_INTERNAL(cl_khr_int64_base_atomics, 100, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_int64_extended_atomics, 100, ~0U)
+OPENCL_COREFEATURE(cl_khr_byte_addressable_store, 100, OCL_C_11P)
+OPENCL_COREFEATURE(cl_khr_global_int32_base_atomics, 100, OCL_C_11P)
+OPENCL_COREFEATURE(cl_khr_global_int32_extended_atomics, 100, OCL_C_11P)
+OPENCL_COREFEATURE(cl_khr_local_int32_base_atomics, 100, OCL_C_11P)
+OPENCL_COREFEATURE(cl_khr_local_int32_extended_atomics, 100, OCL_C_11P)
+OPENCL_OPTIONALCOREFEATURE(cl_khr_fp64, 100, OCL_C_12P)
+OPENCL_EXTENSION(cl_khr_fp16, 100)
+OPENCL_EXTENSION(cl_khr_int64_base_atomics, 100)
+OPENCL_EXTENSION(cl_khr_int64_extended_atomics, 100)
+OPENCL_GENERIC_EXTENSION(cl_khr_3d_image_writes, 100, OCL_C_20, OCL_C_30)
 
 // EMBEDDED_PROFILE
-OPENCLEXT_INTERNAL(cles_khr_int64, 110, ~0U)
+OPENCL_EXTENSION(cles_khr_int64, 110)
 
 // OpenCL 1.2.
-OPENCLEXT_INTERNAL(cl_khr_depth_images, 120, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_gl_msaa_sharing, 120, ~0U)
+OPENCL_EXTENSION(cl_khr_depth_images, 120)
+OPENCL_EXTENSION(cl_khr_gl_msaa_sharing, 120)
 
 // OpenCL 2.0.
-OPENCLEXT_INTERNAL(cl_khr_mipmap_image, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_mipmap_image_writes, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_srgb_image_writes, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroups, 200, ~0U)
+OPENCL_EXTENSION(cl_khr_mipmap_image, 200)
+OPENCL_EXTENSION(cl_khr_mipmap_image_writes, 200)
+OPENCL_EXTENSION(cl_khr_srgb_image_writes, 200)
+OPENCL_EXTENSION(cl_khr_subgroups, 200)
 
 // Clang Extensions.
-OPENCLEXT_INTERNAL(cl_clang_storage_class_specifiers, 100, ~0U)
-OPENCLEXT_INTERNAL(__cl_clang_function_pointers, 100, ~0U)
-OPENCLEXT_INTERNAL(__cl_clang_variadic_functions, 100, ~0U)
+OPENCL_EXTENSION(cl_clang_storage_class_specifiers, 100)
+OPENCL_EXTENSION(__cl_clang_function_pointers, 100)
+OPENCL_EXTENSION(__cl_clang_variadic_functions, 100)
 
 // AMD OpenCL extensions
-OPENCLEXT_INTERNAL(cl_amd_media_ops, 100, ~0U)
-OPENCLEXT_INTERNAL(cl_amd_media_ops2, 100, ~0U)
+OPENCL_EXTENSION(cl_amd_media_ops, 100)
+OPENCL_EXTENSION(cl_amd_media_ops2, 100)
 
 // ARM OpenCL extensions
-OPENCLEXT_INTERNAL(cl_arm_integer_dot_product_int8, 120, ~0U)
-OPENCLEXT_INTERNAL(cl_arm_integer_dot_product_accumulate_int8, 120, ~0U)
-OPENCLEXT_INTERNAL(cl_arm_integer_dot_product_accumulate_int16, 120, ~0U)
-OPENCLEXT_INTERNAL(cl_arm_integer_dot_product_accumulate_saturate_int8, 120, ~0U)
+OPENCL_EXTENSION(cl_arm_integer_dot_product_int8, 120)
+OPENCL_EXTENSION(cl_arm_integer_dot_product_accumulate_int8, 120)
+OPENCL_EXTENSION(cl_arm_integer_dot_product_accumulate_int16, 120)
+OPENCL_EXTENSION(cl_arm_integer_dot_product_accumulate_saturate_int8, 120)
 
 // Intel OpenCL extensions
-OPENCLEXT_INTERNAL(cl_intel_subgroups, 120, ~0U)
-OPENCLEXT_INTERNAL(cl_intel_subgroups_short, 120, ~0U)
-OPENCLEXT_INTERNAL(cl_intel_device_side_avc_motion_estimation, 120, ~0U)
+OPENCL_EXTENSION(cl_intel_subgroups, 120)
+OPENCL_EXTENSION(cl_intel_subgroups_short, 120)
+OPENCL_EXTENSION(cl_intel_device_side_avc_motion_estimation, 120)
+
 
-#undef OPENCLEXT_INTERNAL
+#undef OPENCL_OPTIONALCOREFEATURE
+#undef OPENCL_COREFEATURE
+#undef OPENCL_GENERIC_EXTENSION
 
-#ifdef OPENCLEXT
-#undef OPENCLEXT
+#ifdef OPENCLEXTNAME
+#undef OPENCLEXTNAME
 #endif
diff --git a/clang/include/clang/Basic/OpenCLOptions.h b/clang/include/clang/Basic/OpenCLOptions.h
index 66dd06db5b83..fe27ef19d4d5 100644
--- a/clang/include/clang/Basic/OpenCLOptions.h
+++ b/clang/include/clang/Basic/OpenCLOptions.h
@@ -19,155 +19,145 @@
 
 namespace clang {
 
+namespace {
+// This enum maps OpenCL version(s) into value. These values are used as
+// a mask to indicate in which OpenCL version(s) extension is a core or
+// optional core feature.
+enum OpenCLVersionID : unsigned int {
+  OCL_C_10 = 0x1,
+  OCL_C_11 = 0x2,
+  OCL_C_12 = 0x4,
+  OCL_C_20 = 0x8,
+  OCL_C_30 = 0x10,
+  OCL_C_ALL = 0x1f,
+  OCL_C_11P = OCL_C_ALL ^ OCL_C_10,              // OpenCL C 1.1+
+  OCL_C_12P = OCL_C_ALL ^ (OCL_C_10 | OCL_C_11), // OpenCL C 1.2+
+};
+
+static inline OpenCLVersionID encodeOpenCLVersion(unsigned OpenCLVersion) {
+  switch (OpenCLVersion) {
+  default:
+    llvm_unreachable("Unknown OpenCL version code");
+  case 100:
+    return OCL_C_10;
+  case 110:
+    return OCL_C_11;
+  case 120:
+    return OCL_C_12;
+  case 200:
+    return OCL_C_20;
+  case 300:
+    return OCL_C_30;
+  }
+}
+
+// Simple helper to check if OpenCL C version is contained in a given encoded
+// OpenCL C version mask
+static inline bool isOpenCLVersionIsContainedInMask(const LangOptions &LO,
+                                                    unsigned Mask) {
+  auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
+  OpenCLVersionID Code = encodeOpenCLVersion(CLVer);
+  return Mask & Code;
+}
+} // end anonymous namespace
+
 /// OpenCL supported extensions and optional core features
 class OpenCLOptions {
-  struct Info {
-    bool Supported; // Is this option supported
-    bool Enabled;   // Is this option enabled
-    unsigned Avail; // Option starts to be available in this OpenCL version
-    unsigned Core;  // Option becomes (optional) core feature in this OpenCL
-                    // version
-    Info(bool S = false, bool E = false, unsigned A = 100, unsigned C = ~0U)
-      :Supported(S), Enabled(E), Avail(A), Core(C){}
-  };
-  llvm::StringMap<Info> OptMap;
 public:
-  /// Check if \c Ext is a recognized OpenCL extension.
-  ///
-  /// \param Ext - Extension to look up.
-  /// \returns \c true if \c Ext is known, \c false otherwise.
-  bool isKnown(llvm::StringRef Ext) const {
-    return OptMap.find(Ext) != OptMap.end();
-  }
+  struct OpenCLOptionInfo {
+    // Option starts to be available in this OpenCL version
+    unsigned Avail;
 
-  /// Check if \c Ext is an enabled OpenCL extension.
-  ///
-  /// \param Ext - Extension to look up.
-  /// \returns \c true if \c Ext is known and enabled, \c false otherwise.
-  bool isEnabled(llvm::StringRef Ext) const {
-    auto E = OptMap.find(Ext);
-    return E != OptMap.end() && E->second.Enabled;
-  }
+    // Option becomes core feature in this OpenCL versions
+    unsigned Core;
+
+    // Option becomes optional core feature in this OpenCL versions
+    unsigned Opt;
+
+    // Is this option supported
+    bool Supported = false;
 
-  /// Check if \c Ext is supported as either an extension or an (optional) core
-  /// feature for the given OpenCL version.
-  ///
-  /// \param Ext - Extension to look up.
-  /// \param LO - \c LangOptions specifying the OpenCL version.
-  /// \returns \c true if \c Ext is known and supported, \c false otherwise.
-  bool isSupported(llvm::StringRef Ext, const LangOptions &LO) const {
-    auto E = OptMap.find(Ext);
-    if (E == OptMap.end()) {
-      return false;
+    // Is this option enabled
+    bool Enabled = false;
+
+    OpenCLOptionInfo(unsigned A = 100, unsigned C = 0U, unsigned O = 0U)
+        : Avail(A), Core(C), Opt(O) {}
+
+    bool isCore() const { return Core != 0U; }
+
+    bool isOptionalCore() const { return Opt != 0U; }
+
+    // Is option available in OpenCL version \p LO.
+    bool isAvailableIn(const LangOptions &LO) const {
+      // In C++ mode all extensions should work at least as in v2.0.
+      auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
+      return CLVer >= Avail;
     }
-    // In C++ mode all extensions should work at least as in v2.0.
-    auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
-    auto I = E->getValue();
-    return I.Supported && I.Avail <= CLVer;
-  }
 
-  /// Check if \c Ext is supported as an (optional) OpenCL core features for
-  /// the given OpenCL version.
-  ///
-  /// \param Ext - Extension to look up.
-  /// \param LO - \c LangOptions specifying the OpenCL version.
-  /// \returns \c true if \c Ext is known and supported, \c false otherwise.
-  bool isSupportedCore(llvm::StringRef Ext, const LangOptions &LO) const {
-    auto E = OptMap.find(Ext);
-    if (E == OptMap.end()) {
-      return false;
+    // Is core option in OpenCL version \p LO.
+    bool isCoreIn(const LangOptions &LO) const {
+      return isAvailableIn(LO) && isOpenCLVersionIsContainedInMask(LO, Core);
     }
-    // In C++ mode all extensions should work at least as in v2.0.
-    auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
-    auto I = E->getValue();
-    return I.Supported && I.Avail <= CLVer && I.Core != ~0U && CLVer >= I.Core;
-  }
 
-  /// Check if \c Ext is a supported OpenCL extension for the given OpenCL
-  /// version.
-  ///
-  /// \param Ext - Extension to look up.
-  /// \param LO - \c LangOptions specifying the OpenCL version.
-  /// \returns \c true if \c Ext is known and supported, \c false otherwise.
-  bool isSupportedExtension(llvm::StringRef Ext, const LangOptions &LO) const {
-    auto E = OptMap.find(Ext);
-    if (E == OptMap.end()) {
-      return false;
+    // Is optional core option in OpenCL version \p LO.
+    bool isOptionalCoreIn(const LangOptions &LO) const {
+      return isAvailableIn(LO) && isOpenCLVersionIsContainedInMask(LO, Opt);
     }
-    // In C++ mode all extensions should work at least as in v2.0.
-    auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
-    auto I = E->getValue();
-    return I.Supported && I.Avail <= CLVer && (I.Core == ~0U || CLVer < I.Core);
-  }
+  };
 
-  void enable(llvm::StringRef Ext, bool V = true) {
-    OptMap[Ext].Enabled = V;
-  }
+  bool isKnown(llvm::StringRef Ext) const;
 
-  /// Enable or disable support for OpenCL extensions
-  /// \param Ext name of the extension optionally prefixed with
-  ///        '+' or '-'
-  /// \param V used when \p Ext is not prefixed by '+' or '-'
-  void support(llvm::StringRef Ext, bool V = true) {
-    assert(!Ext.empty() && "Extension is empty.");
-
-    switch (Ext[0]) {
-    case '+':
-      V = true;
-      Ext = Ext.drop_front();
-      break;
-    case '-':
-      V = false;
-      Ext = Ext.drop_front();
-      break;
-    }
+  bool isEnabled(llvm::StringRef Ext) const;
 
-    if (Ext.equals("all")) {
-      supportAll(V);
-      return;
-    }
-    OptMap[Ext].Supported = V;
-  }
+  // Is supported as either an extension or an (optional) core feature for
+  // OpenCL version \p LO.
+  bool isSupported(llvm::StringRef Ext, const LangOptions &LO) const;
 
-  OpenCLOptions(){
-#define OPENCLEXT_INTERNAL(Ext, AvailVer, CoreVer) \
-    OptMap[#Ext].Avail = AvailVer; \
-    OptMap[#Ext].Core = CoreVer;
-#include "clang/Basic/OpenCLExtensions.def"
-  }
+  // Is supported OpenCL core feature for OpenCL version \p LO.
+  // For supported extension, return false.
+  bool isSupportedCore(llvm::StringRef Ext, const LangOptions &LO) const;
 
-  void addSupport(const OpenCLOptions &Opts) {
-    for (auto &I:Opts.OptMap)
-      if (I.second.Supported)
-        OptMap[I.getKey()].Supported = true;
-  }
+  // Is supported optional core OpenCL feature for OpenCL version \p LO.
+  // For supported extension, return false.
+  bool isSupportedOptionalCore(llvm::StringRef Ext,
+                               const LangOptions &LO) const;
 
-  void copy(const OpenCLOptions &Opts) {
-    OptMap = Opts.OptMap;
-  }
+  // Is supported optional core or core OpenCL feature for OpenCL version \p
+  // LO. For supported extension, return false.
+  bool isSupportedCoreOrOptionalCore(llvm::StringRef Ext,
+                                     const LangOptions &LO) const;
 
-  // Turn on or off support of all options.
-  void supportAll(bool On = true) {
-    for (llvm::StringMap<Info>::iterator I = OptMap.begin(),
-         E = OptMap.end(); I != E; ++I)
-      I->second.Supported = On;
-  }
+  // Is supported OpenCL extension for OpenCL version \p LO.
+  // For supported core or optional core feature, return false.
+  bool isSupportedExtension(llvm::StringRef Ext, const LangOptions &LO) const;
 
-  void disableAll() {
-    for (llvm::StringMap<Info>::iterator I = OptMap.begin(),
-         E = OptMap.end(); I != E; ++I)
-      I->second.Enabled = false;
-  }
+  void enable(llvm::StringRef Ext, bool V = true);
 
-  void enableSupportedCore(LangOptions LO) {
-    for (llvm::StringMap<Info>::iterator I = OptMap.begin(), E = OptMap.end();
-         I != E; ++I)
-      if (isSupportedCore(I->getKey(), LO))
-        I->second.Enabled = true;
-  }
+  /// Enable or disable support for OpenCL extensions
+  /// \param Ext name of the extension (not prefixed with '+' or '-')
+  /// \param V value to set for a extension
+  void support(llvm::StringRef Ext, bool V = true);
+
+  OpenCLOptions();
+  OpenCLOptions(const OpenCLOptions &) = default;
+
+  // Set supported options based on target settings and language version
+  void addSupport(const llvm::StringMap<bool> &FeaturesMap,
+                  const LangOptions &Opts);
+
+  // Disable all extensions
+  void disableAll();
+
+  // Enable supported core and optional core features
+  void enableSupportedCore(const LangOptions &LO);
 
   friend class ASTWriter;
   friend class ASTReader;
+
+  using OpenCLOptionInfoMap = llvm::StringMap<OpenCLOptionInfo>;
+
+private:
+  OpenCLOptionInfoMap OptMap;
 };
 
 } // end namespace clang
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index de91ca2ee82e..b782172d93a3 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1438,21 +1438,39 @@ class TargetInfo : public virtual TransferrableTargetInfo,
   /// Set supported OpenCL extensions and optional core features.
   virtual void setSupportedOpenCLOpts() {}
 
+  virtual void supportAllOpenCLOpts(bool V = true) {
+#define OPENCLEXTNAME(Ext) getTargetOpts().OpenCLFeaturesMap[#Ext] = V;
+#include "clang/Basic/OpenCLExtensions.def"
+  }
+
   /// Set supported OpenCL extensions as written on command line
-  virtual void setOpenCLExtensionOpts() {
+  virtual void setCommandLineOpenCLOpts() {
     for (const auto &Ext : getTargetOpts().OpenCLExtensionsAsWritten) {
-      getTargetOpts().SupportedOpenCLOptions.support(Ext);
+      bool IsPrefixed = (Ext[0] == '+' || Ext[0] == '-');
+      std::string Name = IsPrefixed ? Ext.substr(1) : Ext;
+      bool V = IsPrefixed ? Ext[0] == '+' : true;
+
+      if (Name == "all") {
+        supportAllOpenCLOpts(V);
+        continue;
+      }
+
+      getTargetOpts().OpenCLFeaturesMap[Name] = V;
     }
   }
 
+  /// Define OpenCL macros based on target settings and language version
+  void getOpenCLFeatureDefines(const LangOptions &Opts,
+                               MacroBuilder &Builder) const;
+
   /// Get supported OpenCL extensions and optional core features.
-  OpenCLOptions &getSupportedOpenCLOpts() {
-    return getTargetOpts().SupportedOpenCLOptions;
+  llvm::StringMap<bool> &getSupportedOpenCLOpts() {
+    return getTargetOpts().OpenCLFeaturesMap;
   }
 
   /// Get const supported OpenCL extensions and optional core features.
-  const OpenCLOptions &getSupportedOpenCLOpts() const {
-      return getTargetOpts().SupportedOpenCLOptions;
+  const llvm::StringMap<bool> &getSupportedOpenCLOpts() const {
+    return getTargetOpts().OpenCLFeaturesMap;
   }
 
   /// Get address space for OpenCL type.
diff --git a/clang/include/clang/Basic/TargetOptions.h b/clang/include/clang/Basic/TargetOptions.h
index f81c150b7d0a..81c15adb8248 100644
--- a/clang/include/clang/Basic/TargetOptions.h
+++ b/clang/include/clang/Basic/TargetOptions.h
@@ -62,7 +62,7 @@ class TargetOptions {
   llvm::StringMap<bool> FeatureMap;
 
   /// Supported OpenCL extensions and optional core features.
-  OpenCLOptions SupportedOpenCLOptions;
+  llvm::StringMap<bool> OpenCLFeaturesMap;
 
   /// The list of OpenCL extensions to enable or disable, as written on
   /// the command line.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 355e433a42e4..d987b69a76da 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -475,38 +475,23 @@ multiclass BoolOption<string prefix, string spelling_base, KeyPathAndMacro kpm,
                              changed_by_flag, reset_by_flag, flag_suffix>;
 }
 
-/// Creates a BoolOption with the changing available on the CC1 command line.
-multiclass BoolCC1Option<string prefix, string flag_base, KeyPathAndMacro kpm,
-                         DefaultsToBool defaults_to, ChangedBy changed_by,
-                         ResetBy reset_by, BothFlags both = BothFlags<[], "">> {
-  defvar changed_by_cc1
-    = ChangedBy<changed_by.Polarity,
-                !listconcat(changed_by.OptionFlags, [CC1Option]),
-                changed_by.Help, changed_by.ChangedByOptions>;
-
-  defm NAME : BoolOption<prefix, flag_base, kpm, defaults_to, changed_by_cc1,
-                         reset_by, both>;
-}
-
-/// Creates a BoolOption where both of the flags are prefixed with "f", are in
-/// the Group<f_Group>, and the changing flag is also available on the CC1
-/// command line.
+/// Creates a BoolOption where both of the flags are prefixed with "f" and have
+/// the Group<f_Group>.
 multiclass BoolFOption<string flag_base, KeyPathAndMacro kpm,
                        DefaultsToBool defaults_to, ChangedBy changed_by,
                        ResetBy reset_by, BothFlags both = BothFlags<[], "">> {
-  defm NAME : BoolCC1Option<"f", flag_base, kpm, defaults_to, changed_by,
-                            reset_by, both>,
+  defm NAME : BoolOption<"f", flag_base, kpm, defaults_to, changed_by, reset_by,
+                         both>,
               Group<f_Group>;
 }
 
-// Creates a BoolOption where both of the flags are prefixed with "g", are in
-// the Group<g_Group>, and the changing flag is also available on the CC1
-// command line.
+// Creates a BoolOption where both of the flags are prefixed with "g" and have
+// the Group<g_Group>.
 multiclass BoolGOption<string flag_base, KeyPathAndMacro kpm,
                        DefaultsToBool defaults_to, ChangedBy changed_by,
                        ResetBy reset_by, BothFlags both = BothFlags<[], "">> {
-  defm NAME : BoolCC1Option<"g", flag_base, kpm, defaults_to, changed_by,
-                            reset_by, both>,
+  defm NAME : BoolOption<"g", flag_base, kpm, defaults_to, changed_by, reset_by,
+                         both>,
               Group<g_Group>;
 }
 
@@ -971,7 +956,7 @@ defm cuda_approx_transcendentals : OptInFFlag<"cuda-approx-transcendentals", "Us
   " approximate transcendental functions">;
 defm gpu_rdc : BoolFOption<"gpu-rdc",
   LangOpts<"GPURelocatableDeviceCode">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Generate relocatable device code, also known as separate compilation mode">,
+  ChangedBy<PosFlag, [CC1Option], "Generate relocatable device code, also known as separate compilation mode">,
   ResetBy<NegFlag>>;
 def : Flag<["-"], "fcuda-rdc">, Alias<fgpu_rdc>;
 def : Flag<["-"], "fno-cuda-rdc">, Alias<fno_gpu_rdc>;
@@ -991,17 +976,17 @@ def fhip_dump_offload_linker_script : Flag<["-"], "fhip-dump-offload-linker-scri
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>;
 defm hip_new_launch_api : BoolFOption<"hip-new-launch-api",
   LangOpts<"HIPUseNewLaunchAPI">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use">, ResetBy<NegFlag, [], "Don't use">,
+  ChangedBy<PosFlag, [CC1Option], "Use">, ResetBy<NegFlag, [], "Don't use">,
   BothFlags<[], " new kernel launching API for HIP">>;
 defm gpu_allow_device_init : OptInFFlag<"gpu-allow-device-init",
   "Allow", "Don't allow", " device side init function in HIP">;
 defm gpu_defer_diag : BoolFOption<"gpu-defer-diag",
   LangOpts<"GPUDeferDiag">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Defer">, ResetBy<NegFlag, [], "Don't defer">,
+  ChangedBy<PosFlag, [CC1Option], "Defer">, ResetBy<NegFlag, [], "Don't defer">,
   BothFlags<[], " host/device related diagnostic messages for CUDA/HIP">>;
 defm gpu_exclude_wrong_side_overloads : BoolFOption<"gpu-exclude-wrong-side-overloads",
   LangOpts<"GPUExcludeWrongSideOverloads">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Always exclude wrong side overloads">,
+  ChangedBy<PosFlag, [CC1Option], "Always exclude wrong side overloads">,
   ResetBy<NegFlag, [], "Exclude wrong side overloads only if there are same side overloads">,
   BothFlags<[HelpHidden], " in overloading resolution for CUDA/HIP">>;
 def gpu_max_threads_per_block_EQ : Joined<["--"], "gpu-max-threads-per-block=">,
@@ -1063,14 +1048,14 @@ def fPIE : Flag<["-"], "fPIE">, Group<f_Group>;
 def fno_PIE : Flag<["-"], "fno-PIE">, Group<f_Group>;
 defm access_control : BoolFOption<"access-control",
   LangOpts<"AccessControl">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Disable C++ access control">,
+  ChangedBy<NegFlag, [CC1Option], "Disable C++ access control">,
   ResetBy<PosFlag>>;
 def falign_functions : Flag<["-"], "falign-functions">, Group<f_Group>;
 def falign_functions_EQ : Joined<["-"], "falign-functions=">, Group<f_Group>;
 def fno_align_functions: Flag<["-"], "fno-align-functions">, Group<f_Group>;
 defm allow_editor_placeholders : BoolFOption<"allow-editor-placeholders",
   LangOpts<"AllowEditorPlaceholders">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Treat editor placeholders as valid source code">,
+  ChangedBy<PosFlag, [CC1Option], "Treat editor placeholders as valid source code">,
   ResetBy<NegFlag>>;
 def fallow_unsupported : Flag<["-"], "fallow-unsupported">, Group<f_Group>;
 def fapple_kext : Flag<["-"], "fapple-kext">, Group<f_Group>, Flags<[CC1Option]>,
@@ -1078,7 +1063,7 @@ def fapple_kext : Flag<["-"], "fapple-kext">, Group<f_Group>, Flags<[CC1Option]>
   MarshallingInfoFlag<LangOpts<"AppleKext">>;
 defm apple_pragma_pack : BoolFOption<"apple-pragma-pack",
   LangOpts<"ApplePragmaPack">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable Apple gcc-compatible #pragma pack handling">,
+  ChangedBy<PosFlag, [CC1Option], "Enable Apple gcc-compatible #pragma pack handling">,
   ResetBy<NegFlag>>;
 def fxl_pragma_pack : Flag<["-"], "fxl-pragma-pack">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Enable IBM XL #pragma pack handling">;
@@ -1126,7 +1111,7 @@ def fembed_bitcode_marker : Flag<["-"], "fembed-bitcode-marker">,
   HelpText<"Embed placeholder LLVM IR data as a marker">;
 defm gnu_inline_asm : BoolFOption<"gnu-inline-asm",
   LangOpts<"GNUAsm">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Disable GNU style inline asm">, ResetBy<PosFlag>>;
+  ChangedBy<NegFlag, [CC1Option], "Disable GNU style inline asm">, ResetBy<PosFlag>>;
 
 def fprofile_sample_use : Flag<["-"], "fprofile-sample-use">, Group<f_Group>,
     Flags<[CoreOption]>;
@@ -1165,7 +1150,7 @@ def fdebug_compilation_dir_EQ : Joined<["-"], "fdebug-compilation-dir=">,
     Alias<fdebug_compilation_dir>;
 defm debug_info_for_profiling : BoolFOption<"debug-info-for-profiling",
   CodeGenOpts<"DebugInfoForProfiling">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Emit extra debug info to make sample profile more accurate">,
+  ChangedBy<PosFlag, [CC1Option], "Emit extra debug info to make sample profile more accurate">,
   ResetBy<NegFlag>>;
 def fprofile_instr_generate : Flag<["-"], "fprofile-instr-generate">,
     Group<f_Group>, Flags<[CoreOption]>,
@@ -1186,7 +1171,7 @@ def fprofile_remapping_file : Separate<["-"], "fprofile-remapping-file">,
     Group<f_Group>, Flags<[CoreOption]>, Alias<fprofile_remapping_file_EQ>;
 defm coverage_mapping : BoolFOption<"coverage-mapping",
   CodeGenOpts<"CoverageMapping">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Generate coverage mapping to enable code coverage analysis">,
+  ChangedBy<PosFlag, [CC1Option], "Generate coverage mapping to enable code coverage analysis">,
   ResetBy<NegFlag, [], "Disable code coverage analysis">, BothFlags<[CoreOption]>>;
 def fprofile_generate : Flag<["-"], "fprofile-generate">,
     Group<f_Group>, Flags<[CoreOption]>,
@@ -1218,10 +1203,10 @@ def fno_profile_use : Flag<["-"], "fno-profile-use">,
     Alias<fno_profile_instr_use>;
 defm profile_arcs : BoolFOption<"profile-arcs",
   CodeGenOpts<"EmitGcovArcs">, DefaultsToFalse,
-  ChangedBy<PosFlag, [LinkOption]>, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option, LinkOption]>, ResetBy<NegFlag>>;
 defm test_coverage : BoolFOption<"test-coverage",
   CodeGenOpts<"EmitGcovNotes">, DefaultsToFalse,
-  ChangedBy<PosFlag>, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option]>, ResetBy<NegFlag>>;
 def fprofile_filter_files_EQ : Joined<["-"], "fprofile-filter-files=">,
     Group<f_Group>, Flags<[CC1Option, CoreOption]>,
     HelpText<"Instrument only functions from files where names match any regex separated by a semi-colon">,
@@ -1246,20 +1231,20 @@ def forder_file_instrumentation : Flag<["-"], "forder-file-instrumentation">,
 
 defm addrsig : BoolFOption<"addrsig",
   CodeGenOpts<"Addrsig">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Emit">, ResetBy<NegFlag, [], "Don't emit">,
+  ChangedBy<PosFlag, [CC1Option], "Emit">, ResetBy<NegFlag, [], "Don't emit">,
   BothFlags<[CoreOption], " an address-significance table">>;
 defm blocks : OptInFFlag<"blocks", "Enable the 'blocks' language feature", "", "", [CoreOption]>;
 def fbootclasspath_EQ : Joined<["-"], "fbootclasspath=">, Group<f_Group>;
 defm borland_extensions : BoolFOption<"borland-extensions",
   LangOpts<"Borland">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Accept non-standard constructs supported by the Borland compiler">,
+  ChangedBy<PosFlag, [CC1Option], "Accept non-standard constructs supported by the Borland compiler">,
   ResetBy<NegFlag>>;
 def fbuiltin : Flag<["-"], "fbuiltin">, Group<f_Group>, Flags<[CoreOption]>;
 def fbuiltin_module_map : Flag <["-"], "fbuiltin-module-map">, Group<f_Group>,
   Flags<[NoXarchOption]>, HelpText<"Load the clang builtins module map file.">;
 defm caret_diagnostics : BoolFOption<"caret-diagnostics",
   DiagnosticOpts<"ShowCarets">, DefaultsToTrue,
-  ChangedBy<NegFlag>, ResetBy<PosFlag>>;
+  ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag>>;
 def fclang_abi_compat_EQ : Joined<["-"], "fclang-abi-compat=">, Group<f_clang_Group>,
   Flags<[CC1Option]>, MetaVarName<"<version>">, Values<"<major>.<minor>,latest">,
   HelpText<"Attempt to match the ABI of Clang <version>">;
@@ -1300,7 +1285,7 @@ def fcf_runtime_abi_EQ : Joined<["-"], "fcf-runtime-abi=">, Group<f_Group>,
     MarshallingInfoString<LangOpts<"CFRuntime">, "ObjectiveC">, AutoNormalizeEnum;
 defm constant_cfstrings : BoolFOption<"constant-cfstrings",
   LangOpts<"NoConstantCFStrings">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Disable creation of CodeFoundation-type constant strings">,
+  ChangedBy<NegFlag, [CC1Option], "Disable creation of CodeFoundation-type constant strings">,
   ResetBy<PosFlag>>;
 def fconstant_string_class_EQ : Joined<["-"], "fconstant-string-class=">, Group<f_Group>;
 def fconstexpr_depth_EQ : Joined<["-"], "fconstexpr-depth=">, Group<f_Group>;
@@ -1316,7 +1301,7 @@ def fcrash_diagnostics_dir : Joined<["-"], "fcrash-diagnostics-dir=">, Group<f_c
 def fcreate_profile : Flag<["-"], "fcreate-profile">, Group<f_Group>;
 defm cxx_exceptions: BoolFOption<"cxx-exceptions",
   LangOpts<"CXXExceptions">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable C++ exceptions">, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option], "Enable C++ exceptions">, ResetBy<NegFlag>>;
 def fcxx_modules : Flag <["-"], "fcxx-modules">, Group<f_Group>,
   Flags<[NoXarchOption]>;
 def fdebug_pass_arguments : Flag<["-"], "fdebug-pass-arguments">, Group<f_Group>;
@@ -1336,7 +1321,7 @@ def fdiagnostics_print_source_range_info : Flag<["-"], "fdiagnostics-print-sourc
     MarshallingInfoFlag<DiagnosticOpts<"ShowSourceRanges">>;
 defm diagnostics_show_hotness : BoolFOption<"diagnostics-show-hotness",
   CodeGenOpts<"DiagnosticsWithHotness">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable profile hotness information in diagnostic line">,
+  ChangedBy<PosFlag, [CC1Option], "Enable profile hotness information in diagnostic line">,
   ResetBy<NegFlag>>;
 def fdiagnostics_hotness_threshold_EQ : Joined<["-"], "fdiagnostics-hotness-threshold=">,
     Group<f_Group>, Flags<[CC1Option]>, MetaVarName<"<value>">,
@@ -1344,7 +1329,7 @@ def fdiagnostics_hotness_threshold_EQ : Joined<["-"], "fdiagnostics-hotness-thre
     "Use 'auto' to apply the threshold from profile summary">;
 defm diagnostics_show_option : BoolFOption<"diagnostics-show-option",
     DiagnosticOpts<"ShowOptionNames">, DefaultsToTrue,
-    ChangedBy<NegFlag>, ResetBy<PosFlag, [], "Print option name with mappable diagnostics">>;
+    ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag, [], "Print option name with mappable diagnostics">>;
 defm diagnostics_show_note_include_stack : BoolFOption<"diagnostics-show-note-include-stack",
     DiagnosticOpts<"ShowNoteIncludeStack">, DefaultsToFalse,
     ChangedBy<PosFlag, [], "Display include stacks for diagnostic notes">,
@@ -1367,10 +1352,10 @@ def fdwarf2_cfi_asm : Flag<["-"], "fdwarf2-cfi-asm">, Group<clang_ignored_f_Grou
 def fno_dwarf2_cfi_asm : Flag<["-"], "fno-dwarf2-cfi-asm">, Group<clang_ignored_f_Group>;
 defm dwarf_directory_asm : BoolFOption<"dwarf-directory-asm",
   CodeGenOpts<"NoDwarfDirectoryAsm">, DefaultsToFalse,
-  ChangedBy<NegFlag>, ResetBy<PosFlag>>;
+  ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag>>;
 defm elide_constructors : BoolFOption<"elide-constructors",
   LangOpts<"ElideConstructors">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Disable C++ copy constructor elision">,
+  ChangedBy<NegFlag, [CC1Option], "Disable C++ copy constructor elision">,
   ResetBy<PosFlag>>;
 def fno_elide_type : Flag<["-"], "fno-elide-type">, Group<f_Group>,
     Flags<[CC1Option]>,
@@ -1384,13 +1369,13 @@ def femit_all_decls : Flag<["-"], "femit-all-decls">, Group<f_Group>, Flags<[CC1
   MarshallingInfoFlag<LangOpts<"EmitAllDecls">>;
 defm emulated_tls : BoolFOption<"emulated-tls",
   CodeGenOpts<"EmulatedTLS">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use emutls functions to access thread_local variables">,
+  ChangedBy<PosFlag, [CC1Option], "Use emutls functions to access thread_local variables">,
   ResetBy<NegFlag>, BothFlags<[CC1Option]>>;
 def fencoding_EQ : Joined<["-"], "fencoding=">, Group<f_Group>;
 def ferror_limit_EQ : Joined<["-"], "ferror-limit=">, Group<f_Group>, Flags<[CoreOption]>;
 defm exceptions : BoolFOption<"exceptions",
   LangOpts<"Exceptions">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable">, ResetBy<NegFlag, [], "Disable">,
+  ChangedBy<PosFlag, [CC1Option], "Enable">, ResetBy<NegFlag, [], "Disable">,
   BothFlags<[], " support for exception handling">>;
 def fdwarf_exceptions : Flag<["-"], "fdwarf-exceptions">, Group<f_Group>,
   HelpText<"Use DWARF style exceptions">;
@@ -1439,17 +1424,17 @@ def fsignaling_math : Flag<["-"], "fsignaling-math">, Group<f_Group>;
 def fno_signaling_math : Flag<["-"], "fno-signaling-math">, Group<f_Group>;
 defm jump_tables : BoolFOption<"jump-tables",
   CodeGenOpts<"NoUseJumpTables">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Do not use">, ResetBy<PosFlag, [], "Use">,
+  ChangedBy<NegFlag, [CC1Option], "Do not use">, ResetBy<PosFlag, [], "Use">,
   BothFlags<[], " jump tables for lowering switches">>;
 defm force_enable_int128 : OptInFFlag<"force-enable-int128", "Enable", "Disable", " support for int128_t type", [], TargetOpts<"ForceEnableInt128">>;
 defm keep_static_consts : BoolFOption<"keep-static-consts",
   CodeGenOpts<"KeepStaticConsts">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Keep">, ResetBy<NegFlag, [], "Don't keep">,
+  ChangedBy<PosFlag, [CC1Option], "Keep">, ResetBy<NegFlag, [], "Don't keep">,
   BothFlags<[NoXarchOption], " static const variables if unused">>;
 defm fixed_point : OptInFFlag<"fixed-point", "Enable", "Disable", " fixed point types">;
 defm cxx_static_destructors : BoolFOption<"c++-static-destructors",
   LangOpts<"RegisterStaticDestructors">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Disable C++ static destructor registration">,
+  ChangedBy<NegFlag, [CC1Option], "Disable C++ static destructor registration">,
   ResetBy<PosFlag>>;
 def fsymbol_partition_EQ : Joined<["-"], "fsymbol-partition=">, Group<f_Group>,
   Flags<[CC1Option]>, MarshallingInfoString<CodeGenOpts<"SymbolPartition">>;
@@ -1654,7 +1639,7 @@ defm finite_math_only : OptInFFlag<"finite-math-only", "", "", "", [],
   LangOpts<"FiniteMathOnly">, [cl_finite_math_only, ffast_math]>;
 defm signed_zeros : BoolFOption<"signed-zeros",
   LangOpts<"NoSignedZero">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Allow optimizations that ignore the sign of floating point zeros",
+  ChangedBy<NegFlag, [CC1Option], "Allow optimizations that ignore the sign of floating point zeros",
             [cl_no_signed_zeros, menable_unsafe_fp_math]>,
   ResetBy<PosFlag>>;
 def fhonor_nans : Flag<["-"], "fhonor-nans">, Group<f_Group>;
@@ -1681,7 +1666,7 @@ def ffp_contract : Joined<["-"], "ffp-contract=">, Group<f_Group>,
 
 defm strict_float_cast_overflow : BoolFOption<"strict-float-cast-overflow",
   CodeGenOpts<"StrictFloatCastOverflow">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Relax language rules and try to match the behavior"
+  ChangedBy<NegFlag, [CC1Option], "Relax language rules and try to match the behavior"
             " of the target's native float-to-int conversion instructions">,
   ResetBy<PosFlag, [], "Assume that overflowing float-to-int casts are undefined (default)">>;
 
@@ -1690,14 +1675,14 @@ def fno_for_scope : Flag<["-"], "fno-for-scope">, Group<f_Group>;
 
 defm rewrite_imports : BoolFOption<"rewrite-imports",
   PreprocessorOutputOpts<"RewriteImports">, DefaultsToFalse,
-  ChangedBy<PosFlag>, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option]>, ResetBy<NegFlag>>;
 defm rewrite_includes : BoolFOption<"rewrite-includes",
   PreprocessorOutputOpts<"RewriteIncludes">, DefaultsToFalse,
-  ChangedBy<PosFlag>, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option]>, ResetBy<NegFlag>>;
 
 defm delete_null_pointer_checks : BoolFOption<"delete-null-pointer-checks",
   CodeGenOpts<"NullPointerIsValid">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Do not treat usage of null pointers as undefined behavior">,
+  ChangedBy<NegFlag, [CC1Option], "Do not treat usage of null pointers as undefined behavior">,
   ResetBy<PosFlag, [], "Treat usage of null pointers as undefined behavior (default)">,
   BothFlags<[CoreOption]>>;
 
@@ -1711,7 +1696,7 @@ def frewrite_map_file_EQ : Joined<["-"], "frewrite-map-file=">,
 
 defm use_line_directives : BoolFOption<"use-line-directives",
   PreprocessorOutputOpts<"UseLineDirectives">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use #line in preprocessed output">, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option], "Use #line in preprocessed output">, ResetBy<NegFlag>>;
 
 def ffreestanding : Flag<["-"], "ffreestanding">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Assert that the compilation takes place in a freestanding environment">,
@@ -1883,7 +1868,7 @@ def fmacro_backtrace_limit_EQ : Joined<["-"], "fmacro-backtrace-limit=">,
                                 Group<f_Group>, Flags<[NoXarchOption, CoreOption]>;
 defm merge_all_constants : BoolFOption<"merge-all-constants",
   CodeGenOpts<"MergeAllConstants">, DefaultsToFalse,
-  ChangedBy<PosFlag, [CoreOption], "Allow">, ResetBy<NegFlag, [], "Disallow">,
+  ChangedBy<PosFlag, [CC1Option, CoreOption], "Allow">, ResetBy<NegFlag, [], "Disallow">,
   BothFlags<[], " merging of constants">>;
 def fmessage_length_EQ : Joined<["-"], "fmessage-length=">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Format message diagnostics so that they fit within N columns">,
@@ -1906,7 +1891,7 @@ def fms_compatibility_version
                "(default))">;
 defm delayed_template_parsing : BoolFOption<"delayed-template-parsing",
   LangOpts<"DelayedTemplateParsing">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Parse templated function definitions at the end of the translation unit">,
+  ChangedBy<PosFlag, [CC1Option], "Parse templated function definitions at the end of the translation unit">,
   ResetBy<NegFlag, [NoXarchOption], "Disable delayed template parsing">,
   BothFlags<[CoreOption]>>;
 def fms_memptr_rep_EQ : Joined<["-"], "fms-memptr-rep=">, Group<f_Group>, Flags<[CC1Option]>,
@@ -2031,7 +2016,7 @@ def fno_modules_search_all : Flag <["-"], "fno-modules-search-all">, Group<f_Gro
   Flags<[NoXarchOption, CC1Option]>;
 defm implicit_modules : BoolFOption<"implicit-modules",
   LangOpts<"ImplicitModules">, DefaultsToTrue,
-  ChangedBy<NegFlag>, ResetBy<PosFlag>, BothFlags<[NoXarchOption]>>;
+  ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag>, BothFlags<[NoXarchOption]>>;
 def fretain_comments_from_system_headers : Flag<["-"], "fretain-comments-from-system-headers">, Group<f_Group>, Flags<[CC1Option]>,
   MarshallingInfoFlag<LangOpts<"RetainCommentsFromSystemHeaders">>;
 
@@ -2119,7 +2104,7 @@ def fno_temp_file : Flag<["-"], "fno-temp-file">, Group<f_Group>,
   MarshallingInfoNegativeFlag<FrontendOpts<"UseTemporary">>;
 defm use_cxa_atexit : BoolFOption<"use-cxa-atexit",
   CodeGenOpts<"CXAAtExit">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Don't use __cxa_atexit for calling destructors">,
+  ChangedBy<NegFlag, [CC1Option], "Don't use __cxa_atexit for calling destructors">,
   ResetBy<PosFlag>>;
 def fno_unit_at_a_time : Flag<["-"], "fno-unit-at-a-time">, Group<f_Group>;
 def fno_unwind_tables : Flag<["-"], "fno-unwind-tables">, Group<f_Group>;
@@ -2132,7 +2117,7 @@ def fobjc_arc : Flag<["-"], "fobjc-arc">, Group<f_Group>, Flags<[CC1Option]>,
 def fno_objc_arc : Flag<["-"], "fno-objc-arc">, Group<f_Group>;
 defm objc_convert_messages_to_runtime_calls : BoolFOption<"objc-convert-messages-to-runtime-calls",
   CodeGenOpts<"ObjCConvertMessagesToRuntimeCalls">, DefaultsToTrue,
-  ChangedBy<NegFlag>, ResetBy<PosFlag>>;
+  ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag>>;
 defm objc_arc_exceptions : OptInFFlag<"objc-arc-exceptions",
   "Use EH-safe code when synthesizing retains and releases in -fobjc-arc",
   "", "", [], CodeGenOpts<"ObjCAutoRefCountExceptions">>;
@@ -2140,18 +2125,18 @@ def fobjc_atdefs : Flag<["-"], "fobjc-atdefs">, Group<clang_ignored_f_Group>;
 def fobjc_call_cxx_cdtors : Flag<["-"], "fobjc-call-cxx-cdtors">, Group<clang_ignored_f_Group>;
 defm objc_exceptions : BoolFOption<"objc-exceptions",
   LangOpts<"ObjCExceptions">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable Objective-C exceptions">, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option], "Enable Objective-C exceptions">, ResetBy<NegFlag>>;
 defm application_extension : BoolFOption<"application-extension",
   LangOpts<"AppExt">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Restrict code to those available for App Extensions">,
+  ChangedBy<PosFlag, [CC1Option], "Restrict code to those available for App Extensions">,
   ResetBy<NegFlag>>;
 defm relaxed_template_template_args : BoolFOption<"relaxed-template-template-args",
   LangOpts<"RelaxedTemplateTemplateArgs">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable C++17 relaxed template template argument matching">,
+  ChangedBy<PosFlag, [CC1Option], "Enable C++17 relaxed template template argument matching">,
   ResetBy<NegFlag>>;
 defm sized_deallocation : BoolFOption<"sized-deallocation",
   LangOpts<"SizedDeallocation">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable C++14 sized global deallocation functions">,
+  ChangedBy<PosFlag, [CC1Option], "Enable C++14 sized global deallocation functions">,
   ResetBy<NegFlag>>;
 def faligned_allocation : Flag<["-"], "faligned-allocation">, Flags<[CC1Option]>,
   HelpText<"Enable C++17 aligned allocation functions">, Group<f_Group>;
@@ -2170,7 +2155,7 @@ def fobjc_legacy_dispatch : Flag<["-"], "fobjc-legacy-dispatch">, Group<f_Group>
 def fobjc_new_property : Flag<["-"], "fobjc-new-property">, Group<clang_ignored_f_Group>;
 defm objc_infer_related_result_type : BoolFOption<"objc-infer-related-result-type",
   LangOpts<"ObjCInferRelatedResultType">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "do not infer Objective-C related result type based on method family">,
+  ChangedBy<NegFlag, [CC1Option], "do not infer Objective-C related result type based on method family">,
   ResetBy<PosFlag>>;
 def fobjc_link_runtime: Flag<["-"], "fobjc-link-runtime">, Group<f_Group>;
 def fobjc_weak : Flag<["-"], "fobjc-weak">, Group<f_Group>, Flags<[CC1Option]>,
@@ -2225,7 +2210,7 @@ def fopenmp_cuda_teams_reduction_recs_num_EQ : Joined<["-"], "fopenmp-cuda-teams
   Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
 defm openmp_optimistic_collapse : BoolFOption<"openmp-optimistic-collapse",
   LangOpts<"OpenMPOptimisticCollapse">, DefaultsToFalse,
-  ChangedBy<PosFlag>, ResetBy<NegFlag>, BothFlags<[NoArgumentUnused, HelpHidden]>>;
+  ChangedBy<PosFlag, [CC1Option]>, ResetBy<NegFlag>, BothFlags<[NoArgumentUnused, HelpHidden]>>;
 def fopenmp_cuda_parallel_target_regions : Flag<["-"], "fopenmp-cuda-parallel-target-regions">, Group<f_Group>,
   Flags<[CC1Option, NoArgumentUnused, HelpHidden]>,
   HelpText<"Support parallel execution of target regions on Cuda-based devices.">;
@@ -2238,7 +2223,7 @@ def fno_optimize_sibling_calls : Flag<["-"], "fno-optimize-sibling-calls">, Grou
 def foptimize_sibling_calls : Flag<["-"], "foptimize-sibling-calls">, Group<f_Group>;
 defm escaping_block_tail_calls : BoolFOption<"escaping-block-tail-calls",
   CodeGenOpts<"NoEscapingBlockTailCalls">, DefaultsToFalse,
-  ChangedBy<NegFlag>, ResetBy<PosFlag>>;
+  ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag>>;
 def force__cpusubtype__ALL : Flag<["-"], "force_cpusubtype_ALL">;
 def force__flat__namespace : Flag<["-"], "force_flat_namespace">;
 def force__load : Separate<["-"], "force_load">;
@@ -2255,7 +2240,7 @@ def fmax_type_align_EQ : Joined<["-"], "fmax-type-align=">, Group<f_Group>, Flag
 def fno_max_type_align : Flag<["-"], "fno-max-type-align">, Group<f_Group>;
 defm pascal_strings : BoolFOption<"pascal-strings",
   LangOpts<"PascalStrings">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Recognize and construct Pascal-style string literals">,
+  ChangedBy<PosFlag, [CC1Option], "Recognize and construct Pascal-style string literals">,
   ResetBy<NegFlag>>;
 // Note: This flag has different semantics in the driver and in -cc1. The driver accepts -fpatchable-function-entry=M,N
 // and forwards it to -cc1 as -fpatchable-function-entry=M and -fpatchable-function-entry-offset=N. In -cc1, both flags
@@ -2276,15 +2261,15 @@ def fno_direct_access_external_data : Flag<["-"], "fno-direct-access-external-da
   HelpText<"Use GOT indirection to reference external data symbols">;
 defm plt : BoolFOption<"plt",
   CodeGenOpts<"NoPLT">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Use GOT indirection instead of PLT to make external function calls (x86 only)">,
+  ChangedBy<NegFlag, [CC1Option], "Use GOT indirection instead of PLT to make external function calls (x86 only)">,
   ResetBy<PosFlag>>;
 defm ropi : BoolFOption<"ropi",
   LangOpts<"ROPI">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Generate read-only position independent code (ARM only)">,
+  ChangedBy<PosFlag, [CC1Option], "Generate read-only position independent code (ARM only)">,
   ResetBy<NegFlag>>;
 defm rwpi : BoolFOption<"rwpi",
   LangOpts<"RWPI">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Generate read-write position independent code (ARM only)">,
+  ChangedBy<PosFlag, [CC1Option], "Generate read-write position independent code (ARM only)">,
   ResetBy<NegFlag>>;
 def fplugin_EQ : Joined<["-"], "fplugin=">, Group<f_Group>, Flags<[NoXarchOption]>, MetaVarName<"<dsopath>">,
   HelpText<"Load the named plugin (dynamic shared object)">;
@@ -2294,7 +2279,7 @@ def fpass_plugin_EQ : Joined<["-"], "fpass-plugin=">,
   MarshallingInfoStringVector<CodeGenOpts<"PassPlugins">>;
 defm preserve_as_comments : BoolFOption<"preserve-as-comments",
   CodeGenOpts<"PreserveAsmComments">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Do not preserve comments in inline assembly">,
+  ChangedBy<NegFlag, [CC1Option], "Do not preserve comments in inline assembly">,
   ResetBy<PosFlag>>;
 def framework : Separate<["-"], "framework">, Flags<[LinkerInput]>;
 def frandom_seed_EQ : Joined<["-"], "frandom-seed=">, Group<clang_ignored_f_Group>;
@@ -2305,7 +2290,7 @@ defm rtti_data : OptOutFFlag<"rtti-data", "", "Disable generation of RTTI data">
 def : Flag<["-"], "fsched-interblock">, Group<clang_ignored_f_Group>;
 defm short_enums : BoolFOption<"short-enums",
   LangOpts<"ShortEnums">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Allocate to an enum type only as many bytes as it"
+  ChangedBy<PosFlag, [CC1Option], "Allocate to an enum type only as many bytes as it"
            " needs for the declared range of possible values">,
   ResetBy<NegFlag>>;
 def fchar8__t : Flag<["-"], "fchar8_t">, Group<f_Group>, Flags<[CC1Option]>,
@@ -2323,15 +2308,15 @@ def fshow_overloads_EQ : Joined<["-"], "fshow-overloads=">, Group<f_Group>, Flag
   MarshallingInfoString<DiagnosticOpts<"ShowOverloads">, "Ovl_All">, AutoNormalizeEnum;
 defm show_column : BoolFOption<"show-column",
   DiagnosticOpts<"ShowColumn">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Do not include column number on diagnostics">,
+  ChangedBy<NegFlag, [CC1Option], "Do not include column number on diagnostics">,
   ResetBy<PosFlag>>;
 defm show_source_location : BoolFOption<"show-source-location",
   DiagnosticOpts<"ShowLocation">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Do not include source location information with diagnostics">,
+  ChangedBy<NegFlag, [CC1Option], "Do not include source location information with diagnostics">,
   ResetBy<PosFlag>>;
 defm spell_checking : BoolFOption<"spell-checking",
   LangOpts<"SpellChecking">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Disable spell-checking">, ResetBy<PosFlag>>;
+  ChangedBy<NegFlag, [CC1Option], "Disable spell-checking">, ResetBy<PosFlag>>;
 def fspell_checking_limit_EQ : Joined<["-"], "fspell-checking-limit=">, Group<f_Group>;
 def fsigned_bitfields : Flag<["-"], "fsigned-bitfields">, Group<f_Group>;
 defm signed_char : OptOutFFlag<"signed-char", "char is signed", "char is unsigned">;
@@ -2340,7 +2325,7 @@ def fstack_protector_all : Flag<["-"], "fstack-protector-all">, Group<f_Group>,
   HelpText<"Enable stack protectors for all functions">;
 defm stack_clash_protection : BoolFOption<"stack-clash-protection",
   CodeGenOpts<"StackClashProtector">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable">, ResetBy<NegFlag, [], "Disable">,
+  ChangedBy<PosFlag, [CC1Option], "Enable">, ResetBy<NegFlag, [], "Disable">,
   BothFlags<[], " stack clash protection">>;
 def fstack_protector_strong : Flag<["-"], "fstack-protector-strong">, Group<f_Group>,
   HelpText<"Enable stack protectors for some functions vulnerable to stack smashing. "
@@ -2388,7 +2373,7 @@ def fstrict_enums : Flag<["-"], "fstrict-enums">, Group<f_Group>, Flags<[CC1Opti
   MarshallingInfoFlag<CodeGenOpts<"StrictEnums">>;
 defm strict_vtable_pointers : BoolFOption<"strict-vtable-pointers",
   CodeGenOpts<"StrictVTablePointers">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable optimizations based on the strict rules for"
+  ChangedBy<PosFlag, [CC1Option], "Enable optimizations based on the strict rules for"
             " overwriting polymorphic C++ objects">,
   ResetBy<NegFlag>>;
 def fstrict_overflow : Flag<["-"], "fstrict-overflow">, Group<f_Group>;
@@ -2487,7 +2472,7 @@ def Wframe_larger_than_EQ : Joined<["-"], "Wframe-larger-than=">, Group<f_Group>
 def : Flag<["-"], "fterminated-vtables">, Alias<fapple_kext>;
 defm threadsafe_statics : BoolFOption<"threadsafe-statics",
   LangOpts<"ThreadsafeStatics">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Do not emit code to make initialization of local statics thread safe">,
+  ChangedBy<NegFlag, [CC1Option], "Do not emit code to make initialization of local statics thread safe">,
   ResetBy<PosFlag>>;
 def ftime_report : Flag<["-"], "ftime-report">, Group<f_Group>, Flags<[CC1Option]>,
   MarshallingInfoFlag<CodeGenOpts<"TimePasses">>;
@@ -2533,7 +2518,7 @@ def fno_unroll_loops : Flag<["-"], "fno-unroll-loops">, Group<f_Group>,
   HelpText<"Turn off loop unroller">, Flags<[CC1Option]>;
 defm reroll_loops : BoolFOption<"reroll-loops",
   CodeGenOpts<"RerollLoops">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Turn on loop reroller">, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option], "Turn on loop reroller">, ResetBy<NegFlag>>;
 def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>,
   HelpText<"Process trigraph sequences">, Flags<[CC1Option]>;
 def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>,
@@ -2544,11 +2529,11 @@ def fno_unsigned_char : Flag<["-"], "fno-unsigned-char">;
 def funwind_tables : Flag<["-"], "funwind-tables">, Group<f_Group>;
 defm register_global_dtors_with_atexit : BoolFOption<"register-global-dtors-with-atexit",
   CodeGenOpts<"RegisterGlobalDtorsWithAtExit">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use">, ResetBy<NegFlag, [], "Don't use">,
+  ChangedBy<PosFlag, [CC1Option], "Use">, ResetBy<NegFlag, [], "Don't use">,
   BothFlags<[], " atexit or __cxa_atexit to register global destructors">>;
 defm use_init_array : BoolFOption<"use-init-array",
   CodeGenOpts<"UseInitArray">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Use .ctors/.dtors instead of .init_array/.fini_array">,
+  ChangedBy<NegFlag, [CC1Option], "Use .ctors/.dtors instead of .init_array/.fini_array">,
   ResetBy<PosFlag>>;
 def fno_var_tracking : Flag<["-"], "fno-var-tracking">, Group<clang_ignored_f_Group>;
 def fverbose_asm : Flag<["-"], "fverbose-asm">, Group<f_Group>,
@@ -2556,7 +2541,7 @@ def fverbose_asm : Flag<["-"], "fverbose-asm">, Group<f_Group>,
 def dA : Flag<["-"], "dA">, Alias<fverbose_asm>;
 defm visibility_from_dllstorageclass : BoolFOption<"visibility-from-dllstorageclass",
   LangOpts<"VisibilityFromDLLStorageClass">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Set the visiblity of symbols in the generated code from their DLL storage class">,
+  ChangedBy<PosFlag, [CC1Option], "Set the visiblity of symbols in the generated code from their DLL storage class">,
   ResetBy<NegFlag>>;
 def fvisibility_dllexport_EQ : Joined<["-"], "fvisibility-dllexport=">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"The visibility for dllexport defintions [-fvisibility-from-dllstorageclass]">,
@@ -2581,7 +2566,7 @@ def fvisibility_inlines_hidden : Flag<["-"], "fvisibility-inlines-hidden">, Grou
   Flags<[CC1Option]>, MarshallingInfoFlag<LangOpts<"InlineVisibilityHidden">>;
 defm visibility_inlines_hidden_static_local_var : BoolFOption<"visibility-inlines-hidden-static-local-var",
   LangOpts<"VisibilityInlinesHiddenStaticLocalVar">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "When -fvisibility-inlines-hidden is enabled, static variables in"
+  ChangedBy<PosFlag, [CC1Option], "When -fvisibility-inlines-hidden is enabled, static variables in"
             " inline C++ member functions will also be given hidden visibility by default">,
   ResetBy<NegFlag, [], "Disables -fvisibility-inlines-hidden-static-local-var"
          " (this is the default on non-darwin targets)">, BothFlags<[CC1Option]>>;
@@ -2593,19 +2578,19 @@ def fvisibility_global_new_delete_hidden : Flag<["-"], "fvisibility-global-new-d
   MarshallingInfoFlag<LangOpts<"GlobalAllocationFunctionVisibilityHidden">>;
 defm whole_program_vtables : BoolFOption<"whole-program-vtables",
   CodeGenOpts<"WholeProgramVTables">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enables whole-program vtable optimization. Requires -flto">,
+  ChangedBy<PosFlag, [CC1Option], "Enables whole-program vtable optimization. Requires -flto">,
   ResetBy<NegFlag>, BothFlags<[CoreOption]>>;
 defm split_lto_unit : BoolFOption<"split-lto-unit",
   CodeGenOpts<"EnableSplitLTOUnit">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enables splitting of the LTO unit">,
+  ChangedBy<PosFlag, [CC1Option], "Enables splitting of the LTO unit">,
   ResetBy<NegFlag>, BothFlags<[CoreOption]>>;
 defm force_emit_vtables : BoolFOption<"force-emit-vtables",
   CodeGenOpts<"ForceEmitVTables">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Emits more virtual tables to improve devirtualization">,
+  ChangedBy<PosFlag, [CC1Option], "Emits more virtual tables to improve devirtualization">,
   ResetBy<NegFlag>, BothFlags<[CoreOption]>>;
 defm virtual_function_elimination : BoolFOption<"virtual-function-elimination",
   CodeGenOpts<"VirtualFunctionElimination">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enables dead virtual function elimination optimization. Requires -flto=full">,
+  ChangedBy<PosFlag, [CC1Option], "Enables dead virtual function elimination optimization. Requires -flto=full">,
   ResetBy<NegFlag>, BothFlags<[CoreOption]>>;
 
 def fwrapv : Flag<["-"], "fwrapv">, Group<f_Group>, Flags<[CC1Option]>,
@@ -2615,7 +2600,7 @@ def fwritable_strings : Flag<["-"], "fwritable-strings">, Group<f_Group>, Flags<
   MarshallingInfoFlag<LangOpts<"WritableStrings">>;
 defm zero_initialized_in_bss : BoolFOption<"zero-initialized-in-bss",
   CodeGenOpts<"NoZeroInitializedInBSS">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Don't place zero initialized data in BSS">,
+  ChangedBy<NegFlag, [CC1Option], "Don't place zero initialized data in BSS">,
   ResetBy<PosFlag>>;
 defm function_sections : OptInFFlag<"function-sections", "Place each function in its own section">;
 def fbasic_block_sections_EQ : Joined<["-"], "fbasic-block-sections=">, Group<f_Group>,
@@ -2626,34 +2611,34 @@ def fbasic_block_sections_EQ : Joined<["-"], "fbasic-block-sections=">, Group<f_
   MarshallingInfoString<CodeGenOpts<"BBSections">, [{"none"}]>;
 defm data_sections : BoolFOption<"data-sections",
   CodeGenOpts<"DataSections">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Place each data in its own section">, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option], "Place each data in its own section">, ResetBy<NegFlag>>;
 defm stack_size_section : BoolFOption<"stack-size-section",
   CodeGenOpts<"StackSizeSection">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Emit section containing metadata on function stack sizes">,
+  ChangedBy<PosFlag, [CC1Option], "Emit section containing metadata on function stack sizes">,
   ResetBy<NegFlag>>;
 
 defm unique_basic_block_section_names : BoolFOption<"unique-basic-block-section-names",
   CodeGenOpts<"UniqueBasicBlockSectionNames">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use unique names for basic block sections (ELF Only)">,
+  ChangedBy<PosFlag, [CC1Option], "Use unique names for basic block sections (ELF Only)">,
   ResetBy<NegFlag>>;
 defm unique_internal_linkage_names : BoolFOption<"unique-internal-linkage-names",
   CodeGenOpts<"UniqueInternalLinkageNames">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Uniqueify Internal Linkage Symbol Names by appending"
+  ChangedBy<PosFlag, [CC1Option], "Uniqueify Internal Linkage Symbol Names by appending"
             " the MD5 hash of the module path">,
   ResetBy<NegFlag>>;
 defm unique_section_names : BoolFOption<"unique-section-names",
   CodeGenOpts<"UniqueSectionNames">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Don't use unique names for text and data sections">,
+  ChangedBy<NegFlag, [CC1Option], "Don't use unique names for text and data sections">,
   ResetBy<PosFlag>>;
 
 defm split_machine_functions: BoolFOption<"split-machine-functions",
   CodeGenOpts<"SplitMachineFunctions">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable">, ResetBy<NegFlag, [], "Disable">,
+  ChangedBy<PosFlag, [CC1Option], "Enable">, ResetBy<NegFlag, [], "Disable">,
   BothFlags<[], " late function splitting using profile information (x86 ELF)">>;
 
 defm strict_return : BoolFOption<"strict-return",
   CodeGenOpts<"StrictReturn">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Don't treat control flow paths that fall off the end"
+  ChangedBy<NegFlag, [CC1Option], "Don't treat control flow paths that fall off the end"
             " of a non-void function as unreachable">,
   ResetBy<PosFlag>>;
 
@@ -2668,11 +2653,11 @@ def fdebug_types_section: Flag <["-"], "fdebug-types-section">, Group<f_Group>,
 def fno_debug_types_section: Flag<["-"], "fno-debug-types-section">, Group<f_Group>;
 defm debug_ranges_base_address : BoolFOption<"debug-ranges-base-address",
   CodeGenOpts<"DebugRangesBaseAddress">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use DWARF base address selection entries in .debug_ranges">,
+  ChangedBy<PosFlag, [CC1Option], "Use DWARF base address selection entries in .debug_ranges">,
   ResetBy<NegFlag>>;
 defm split_dwarf_inlining : BoolFOption<"split-dwarf-inlining",
   CodeGenOpts<"SplitDwarfInlining">, DefaultsToTrue,
-  ChangedBy<NegFlag>,
+  ChangedBy<NegFlag, [CC1Option]>,
   ResetBy<PosFlag, [], "Provide minimal debug info in the object/executable"
           " to facilitate online symbolication/stack traces in the absence of"
           " .dwo/.dwp files when using Split DWARF">>;
@@ -2690,7 +2675,7 @@ def fmacro_prefix_map_EQ
     HelpText<"remap file source paths in predefined preprocessor macros">;
 defm force_dwarf_frame : BoolFOption<"force-dwarf-frame",
   CodeGenOpts<"ForceDwarfFrameSection">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Always emit a debug frame section">, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option], "Always emit a debug frame section">, ResetBy<NegFlag>>;
 def g_Flag : Flag<["-"], "g">, Group<g_Group>,
   HelpText<"Generate source-level debug information">;
 def gline_tables_only : Flag<["-"], "gline-tables-only">, Group<gN_Group>,
@@ -2737,7 +2722,7 @@ defm codeview_ghash : BoolOption<"g", "codeview-ghash",
   ResetBy<NegFlag>, BothFlags<[CoreOption]>>;
 defm inline_line_tables : BoolGOption<"inline-line-tables",
   CodeGenOpts<"NoInlineLineTables">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Don't emit inline line tables.">,
+  ChangedBy<NegFlag, [CC1Option], "Don't emit inline line tables.">,
   ResetBy<PosFlag>, BothFlags<[CoreOption]>>;
 
 def gfull : Flag<["-"], "gfull">, Group<g_Group>;
@@ -2755,9 +2740,9 @@ def : Flag<["-"], "grecord-gcc-switches">, Alias<grecord_command_line>;
 def : Flag<["-"], "gno-record-gcc-switches">, Alias<gno_record_command_line>;
 def gstrict_dwarf : Flag<["-"], "gstrict-dwarf">, Group<g_flags_Group>;
 def gno_strict_dwarf : Flag<["-"], "gno-strict-dwarf">, Group<g_flags_Group>;
-defm column_info : BoolCC1Option<"g", "column-info",
+defm column_info : BoolOption<"g", "column-info",
   CodeGenOpts<"DebugColumnInfo">, DefaultsToTrue,
-  ChangedBy<NegFlag>, ResetBy<PosFlag>, BothFlags<[CoreOption]>>,
+  ChangedBy<NegFlag, [CC1Option]>, ResetBy<PosFlag>, BothFlags<[CoreOption]>>,
   Group<g_flags_Group>;
 def gsplit_dwarf : Flag<["-"], "gsplit-dwarf">, Group<g_flags_Group>;
 def gsplit_dwarf_EQ : Joined<["-"], "gsplit-dwarf=">, Group<g_flags_Group>,
@@ -3154,9 +3139,9 @@ def mwavefrontsize64 : Flag<["-"], "mwavefrontsize64">, Group<m_Group>,
 def mno_wavefrontsize64 : Flag<["-"], "mno-wavefrontsize64">, Group<m_Group>,
   HelpText<"Specify wavefront size 32 mode (AMDGPU only)">;
 
-defm unsafe_fp_atomics : BoolCC1Option<"m", "unsafe-fp-atomics",
+defm unsafe_fp_atomics : BoolOption<"m", "unsafe-fp-atomics",
   TargetOpts<"AllowAMDGPUUnsafeFPAtomics">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable unsafe floating point atomic instructions (AMDGPU only)">,
+  ChangedBy<PosFlag, [CC1Option], "Enable unsafe floating point atomic instructions (AMDGPU only)">,
   ResetBy<NegFlag>>, Group<m_Group>;
 
 def faltivec : Flag<["-"], "faltivec">, Group<f_Group>, Flags<[NoXarchOption]>;
@@ -3242,7 +3227,7 @@ def mno_vx : Flag<["-"], "mno-vx">, Group<m_Group>;
 
 defm zvector : BoolFOption<"zvector",
   LangOpts<"ZVector">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Enable System z vector language extension">,
+  ChangedBy<PosFlag, [CC1Option], "Enable System z vector language extension">,
   ResetBy<NegFlag>>;
 def mzvector : Flag<["-"], "mzvector">, Alias<fzvector>;
 def mno_zvector : Flag<["-"], "mno-zvector">, Alias<fno_zvector>;
@@ -3697,7 +3682,7 @@ def y : Joined<["-"], "y">;
 
 defm integrated_as : BoolFOption<"integrated-as",
   CodeGenOpts<"DisableIntegratedAS">, DefaultsToFalse,
-  ChangedBy<NegFlag, [], "Disable">, ResetBy<PosFlag, [], "Enable">,
+  ChangedBy<NegFlag, [CC1Option], "Disable">, ResetBy<PosFlag, [], "Enable">,
   BothFlags<[], " the integrated assembler">>;
 
 def fintegrated_cc1 : Flag<["-"], "fintegrated-cc1">,
@@ -4140,7 +4125,7 @@ defm ipa_cp : BooleanFFlag<"ipa-cp">,
 defm ivopts : BooleanFFlag<"ivopts">, Group<clang_ignored_gcc_optimization_f_Group>;
 defm semantic_interposition : BoolFOption<"semantic-interposition",
   LangOpts<"SemanticInterposition">, DefaultsToFalse,
-  ChangedBy<PosFlag>, ResetBy<NegFlag>>;
+  ChangedBy<PosFlag, [CC1Option]>, ResetBy<NegFlag>>;
 defm non_call_exceptions : BooleanFFlag<"non-call-exceptions">, Group<clang_ignored_f_Group>;
 defm peel_loops : BooleanFFlag<"peel-loops">, Group<clang_ignored_gcc_optimization_f_Group>;
 defm permissive : BooleanFFlag<"permissive">, Group<clang_ignored_f_Group>;
@@ -4785,9 +4770,9 @@ def flto_visibility_public_std:
     Flag<["-"], "flto-visibility-public-std">,
     HelpText<"Use public LTO visibility for classes in std and stdext namespaces">,
     MarshallingInfoFlag<CodeGenOpts<"LTOVisibilityPublicStd">>;
-defm lto_unit : BoolCC1Option<"f", "lto-unit",
+defm lto_unit : BoolOption<"f", "lto-unit",
   CodeGenOpts<"LTOUnit">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Emit IR to support LTO unit features (CFI, whole program vtable opt)">,
+  ChangedBy<PosFlag, [CC1Option], "Emit IR to support LTO unit features (CFI, whole program vtable opt)">,
   ResetBy<NegFlag>>;
 defm debug_pass_manager : BoolOption<"f", "debug-pass-manager",
   CodeGenOpts<"DebugPassManager">, DefaultsToFalse,
@@ -5280,9 +5265,9 @@ def fconstexpr_steps : Separate<["-"], "fconstexpr-steps">,
 def fbracket_depth : Separate<["-"], "fbracket-depth">,
   HelpText<"Maximum nesting level for parentheses, brackets, and braces">,
   MarshallingInfoStringInt<LangOpts<"BracketDepth">, "256">;
-defm const_strings : BoolCC1Option<"f", "const-strings",
+defm const_strings : BoolOption<"f", "const-strings",
   LangOpts<"ConstStrings">, DefaultsToFalse,
-  ChangedBy<PosFlag, [], "Use">, ResetBy<NegFlag, [], "Don't use">,
+  ChangedBy<PosFlag, [CC1Option], "Use">, ResetBy<NegFlag, [], "Don't use">,
   BothFlags<[], " a const qualified type for string literals in C and ObjC">>;
 def fno_bitfield_type_align : Flag<["-"], "fno-bitfield-type-align">,
   HelpText<"Ignore bit-field types when aligning structures">,
@@ -5344,9 +5329,9 @@ def fwchar_type_EQ : Joined<["-"], "fwchar-type=">,
   HelpText<"Select underlying type for wchar_t">, Values<"char,short,int">,
   NormalizedValues<["1", "2", "4"]>,
   MarshallingInfoString<LangOpts<"WCharSize">, "0">, AutoNormalizeEnum;
-defm signed_wchar : BoolCC1Option<"f", "signed-wchar",
+defm signed_wchar : BoolOption<"f", "signed-wchar",
   LangOpts<"WCharIsSigned">, DefaultsToTrue,
-  ChangedBy<NegFlag, [], "Use an unsigned">, ResetBy<PosFlag, [], "Use a signed">,
+  ChangedBy<NegFlag, [CC1Option], "Use an unsigned">, ResetBy<PosFlag, [], "Use a signed">,
   BothFlags<[], " type for wchar_t">>;
 def fcompatibility_qualified_id_block_param_type_checking : Flag<["-"], "fcompatibility-qualified-id-block-type-checking">,
   HelpText<"Allow using blocks with parameters of more specific type than "
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 385803700d58..fcc38e25542f 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -84,10 +84,23 @@ struct FormatStyle {
   /// brackets.
   BracketAlignmentStyle AlignAfterOpenBracket;
 
-  /// \brief If ``true``, aligns consecutive C/C++ preprocessor macros.
+  /// Styles for alignment of consecutive tokens. Tokens can be assignment signs
+  /// (see
+  /// ``AlignConsecutiveAssignments``), bitfield member separators (see
+  /// ``AlignConsecutiveBitFields``), names in declarations (see
+  /// ``AlignConsecutiveDeclarations``) or macro definitions (see
+  /// ``AlignConsecutiveMacros``).
+  enum AlignConsecutiveStyle {
+    ACS_None,
+    ACS_Consecutive,
+    ACS_AcrossEmptyLines,
+    ACS_AcrossComments,
+    ACS_AcrossEmptyLinesAndComments
+  };
+
+  /// Style of aligning consecutive macro definitions.
   ///
-  /// This will align C/C++ preprocessor macros of consecutive lines.
-  /// Will result in formattings like
+  /// ``Consecutive`` will result in formattings like:
   /// \code
   ///   #define SHORT_NAME       42
   ///   #define LONGER_NAME      0x007f
@@ -95,40 +108,271 @@ struct FormatStyle {
   ///   #define foo(x)           (x * x)
   ///   #define bar(y, z)        (y + z)
   /// \endcode
-  bool AlignConsecutiveMacros;
-
-  /// If ``true``, aligns consecutive assignments.
   ///
-  /// This will align the assignment operators of consecutive lines. This
-  /// will result in formattings like
-  /// \code
-  ///   int aaaa = 12;
-  ///   int b    = 23;
-  ///   int ccc  = 23;
-  /// \endcode
-  bool AlignConsecutiveAssignments;
-
-  /// If ``true``, aligns consecutive bitfield members.
-  ///
-  /// This will align the bitfield separators of consecutive lines. This
-  /// will result in formattings like
+  /// Possible values:
+  ///
+  /// * ``ACS_None`` (in configuration: ``None``)
+  ///    Do not align macro definitions on consecutive lines.
+  ///
+  /// * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+  ///    Align macro definitions on consecutive lines. This will result in
+  ///    formattings like:
+  ///    \code
+  ///      #define SHORT_NAME       42
+  ///      #define LONGER_NAME      0x007f
+  ///      #define EVEN_LONGER_NAME (2)
+  ///
+  ///      #define foo(x) (x * x)
+  ///      /* some comment */
+  ///      #define bar(y, z) (y + z)
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+  ///    Same as ACS_Consecutive, but also spans over empty lines, e.g.
+  ///    \code
+  ///      #define SHORT_NAME       42
+  ///      #define LONGER_NAME      0x007f
+  ///      #define EVEN_LONGER_NAME (2)
+  ///
+  ///      #define foo(x)           (x * x)
+  ///      /* some comment */
+  ///      #define bar(y, z) (y + z)
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments, e.g.
+  ///    \code
+  ///      #define SHORT_NAME       42
+  ///      #define LONGER_NAME      0x007f
+  ///      #define EVEN_LONGER_NAME (2)
+  ///
+  ///      #define foo(x)    (x * x)
+  ///      /* some comment */
+  ///      #define bar(y, z) (y + z)
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLinesAndComments``
+  ///   (in configuration: ``AcrossEmptyLinesAndComments``)
+  ///
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments and empty lines, e.g.
+  ///    \code
+  ///      #define SHORT_NAME       42
+  ///      #define LONGER_NAME      0x007f
+  ///      #define EVEN_LONGER_NAME (2)
+  ///
+  ///      #define foo(x)           (x * x)
+  ///      /* some comment */
+  ///      #define bar(y, z)        (y + z)
+  ///    \endcode
+  AlignConsecutiveStyle AlignConsecutiveMacros;
+
+  /// Style of aligning consecutive assignments.
+  ///
+  /// ``Consecutive`` will result in formattings like:
+  /// \code
+  ///   int a            = 1;
+  ///   int somelongname = 2;
+  ///   double c         = 3;
+  /// \endcode
+  ///
+  /// Possible values:
+  ///
+  /// * ``ACS_None`` (in configuration: ``None``)
+  ///    Do not align assignments on consecutive lines.
+  ///
+  /// * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+  ///    Align assignments on consecutive lines. This will result in
+  ///    formattings like:
+  ///    \code
+  ///      int a            = 1;
+  ///      int somelongname = 2;
+  ///      double c         = 3;
+  ///
+  ///      int d = 3;
+  ///      /* A comment. */
+  ///      double e = 4;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+  ///    Same as ACS_Consecutive, but also spans over empty lines, e.g.
+  ///    \code
+  ///      int a            = 1;
+  ///      int somelongname = 2;
+  ///      double c         = 3;
+  ///
+  ///      int d            = 3;
+  ///      /* A comment. */
+  ///      double e = 4;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments, e.g.
+  ///    \code
+  ///      int a            = 1;
+  ///      int somelongname = 2;
+  ///      double c         = 3;
+  ///
+  ///      int d    = 3;
+  ///      /* A comment. */
+  ///      double e = 4;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLinesAndComments``
+  ///   (in configuration: ``AcrossEmptyLinesAndComments``)
+  ///
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments and empty lines, e.g.
+  ///    \code
+  ///      int a            = 1;
+  ///      int somelongname = 2;
+  ///      double c         = 3;
+  ///
+  ///      int d            = 3;
+  ///      /* A comment. */
+  ///      double e         = 4;
+  ///    \endcode
+  AlignConsecutiveStyle AlignConsecutiveAssignments;
+
+  /// Style of aligning consecutive bit field.
+  ///
+  /// ``Consecutive`` will align the bitfield separators of consecutive lines.
+  /// This will result in formattings like:
   /// \code
   ///   int aaaa : 1;
   ///   int b    : 12;
   ///   int ccc  : 8;
   /// \endcode
-  bool AlignConsecutiveBitFields;
-
-  /// If ``true``, aligns consecutive declarations.
   ///
-  /// This will align the declaration names of consecutive lines. This
-  /// will result in formattings like
+  /// Possible values:
+  ///
+  /// * ``ACS_None`` (in configuration: ``None``)
+  ///    Do not align bit fields on consecutive lines.
+  ///
+  /// * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+  ///    Align bit fields on consecutive lines. This will result in
+  ///    formattings like:
+  ///    \code
+  ///      int aaaa : 1;
+  ///      int b    : 12;
+  ///      int ccc  : 8;
+  ///
+  ///      int d : 2;
+  ///      /* A comment. */
+  ///      int ee : 3;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+  ///    Same as ACS_Consecutive, but also spans over empty lines, e.g.
+  ///    \code
+  ///      int aaaa : 1;
+  ///      int b    : 12;
+  ///      int ccc  : 8;
+  ///
+  ///      int d    : 2;
+  ///      /* A comment. */
+  ///      int ee : 3;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments, e.g.
+  ///    \code
+  ///      int aaaa : 1;
+  ///      int b    : 12;
+  ///      int ccc  : 8;
+  ///
+  ///      int d  : 2;
+  ///      /* A comment. */
+  ///      int ee : 3;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLinesAndComments``
+  ///   (in configuration: ``AcrossEmptyLinesAndComments``)
+  ///
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments and empty lines, e.g.
+  ///    \code
+  ///      int aaaa : 1;
+  ///      int b    : 12;
+  ///      int ccc  : 8;
+  ///
+  ///      int d    : 2;
+  ///      /* A comment. */
+  ///      int ee   : 3;
+  ///    \endcode
+  AlignConsecutiveStyle AlignConsecutiveBitFields;
+
+  /// Style of aligning consecutive declarations.
+  ///
+  /// ``Consecutive`` will align the declaration names of consecutive lines.
+  /// This will result in formattings like:
   /// \code
   ///   int         aaaa = 12;
   ///   float       b = 23;
-  ///   std::string ccc = 23;
-  /// \endcode
-  bool AlignConsecutiveDeclarations;
+  ///   std::string ccc;
+  /// \endcode
+  ///
+  /// Possible values:
+  ///
+  /// * ``ACS_None`` (in configuration: ``None``)
+  ///    Do not align bit declarations on consecutive lines.
+  ///
+  /// * ``ACS_Consecutive`` (in configuration: ``Consecutive``)
+  ///    Align declarations on consecutive lines. This will result in
+  ///    formattings like:
+  ///    \code
+  ///      int         aaaa = 12;
+  ///      float       b = 23;
+  ///      std::string ccc;
+  ///
+  ///      int a = 42;
+  ///      /* A comment. */
+  ///      bool c = false;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLines`` (in configuration: ``AcrossEmptyLines``)
+  ///    Same as ACS_Consecutive, but also spans over empty lines, e.g.
+  ///    \code
+  ///      int         aaaa = 12;
+  ///      float       b = 23;
+  ///      std::string ccc;
+  ///
+  ///      int         a = 42;
+  ///      /* A comment. */
+  ///      bool c = false;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossComments`` (in configuration: ``AcrossComments``)
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments, e.g.
+  ///    \code
+  ///      int         aaaa = 12;
+  ///      float       b = 23;
+  ///      std::string ccc;
+  ///
+  ///      int  a = 42;
+  ///      /* A comment. */
+  ///      bool c = false;
+  ///    \endcode
+  ///
+  /// * ``ACS_AcrossEmptyLinesAndComments``
+  ///   (in configuration: ``AcrossEmptyLinesAndComments``)
+  ///
+  ///    Same as ACS_Consecutive, but also spans over lines only containing
+  ///    comments and empty lines, e.g.
+  ///    \code
+  ///      int         aaaa = 12;
+  ///      float       b = 23;
+  ///      std::string ccc;
+  ///
+  ///      int         a = 42;
+  ///      /* A comment. */
+  ///      bool        c = false;
+  ///    \endcode
+  AlignConsecutiveStyle AlignConsecutiveDeclarations;
 
   /// Different styles for aligning escaped newlines.
   enum EscapedNewlineAlignmentStyle : unsigned char {
@@ -2725,6 +2969,7 @@ struct FormatStyle {
            AlignConsecutiveAssignments == R.AlignConsecutiveAssignments &&
            AlignConsecutiveBitFields == R.AlignConsecutiveBitFields &&
            AlignConsecutiveDeclarations == R.AlignConsecutiveDeclarations &&
+           AlignConsecutiveMacros == R.AlignConsecutiveMacros &&
            AlignEscapedNewlines == R.AlignEscapedNewlines &&
            AlignOperands == R.AlignOperands &&
            AlignTrailingComments == R.AlignTrailingComments &&
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 7e2484581618..5602a3f1d4ae 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1827,8 +1827,7 @@ template <typename DeclT>
 static SourceLocation getTemplateOrInnerLocStart(const DeclT *decl) {
   if (decl->getNumTemplateParameterLists() > 0)
     return decl->getTemplateParameterList(0)->getTemplateLoc();
-  else
-    return decl->getInnerLocStart();
+  return decl->getInnerLocStart();
 }
 
 SourceLocation DeclaratorDecl::getTypeSpecStartLoc() const {
@@ -2136,10 +2135,9 @@ VarDecl::isThisDeclarationADefinition(ASTContext &C) const {
                     TSK_ExplicitSpecialization) ||
          isa<VarTemplatePartialSpecializationDecl>(this)))
       return Definition;
-    else if (!isOutOfLine() && isInline())
+    if (!isOutOfLine() && isInline())
       return Definition;
-    else
-      return DeclarationOnly;
+    return DeclarationOnly;
   }
   // C99 6.7p5:
   //   A definition of an identifier is a declaration for that identifier that
@@ -2202,7 +2200,7 @@ VarDecl *VarDecl::getActingDefinition() {
     Kind = I->isThisDeclarationADefinition();
     if (Kind == Definition)
       return nullptr;
-    else if (Kind == TentativeDefinition)
+    if (Kind == TentativeDefinition)
       LastTentative = I;
   }
   return LastTentative;
@@ -2274,8 +2272,7 @@ VarDecl *VarDecl::getInitializingDeclaration() {
     if (I->isThisDeclarationADefinition()) {
       if (isStaticDataMember())
         return I;
-      else
-        Def = I;
+      Def = I;
     }
   }
   return Def;
@@ -3599,8 +3596,7 @@ bool FunctionDecl::isInlineDefinitionExternallyVisible() const {
 OverloadedOperatorKind FunctionDecl::getOverloadedOperator() const {
   if (getDeclName().getNameKind() == DeclarationName::CXXOperatorName)
     return getDeclName().getCXXOverloadedOperator();
-  else
-    return OO_None;
+  return OO_None;
 }
 
 /// getLiteralIdentifier - The literal suffix identifier this function
@@ -3608,8 +3604,7 @@ OverloadedOperatorKind FunctionDecl::getOverloadedOperator() const {
 const IdentifierInfo *FunctionDecl::getLiteralIdentifier() const {
   if (getDeclName().getNameKind() == DeclarationName::CXXLiteralOperatorName)
     return getDeclName().getCXXLiteralIdentifier();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 FunctionDecl::TemplatedKind FunctionDecl::getTemplatedKind() const {
@@ -3942,8 +3937,8 @@ SourceLocation FunctionDecl::getPointOfInstantiation() const {
         = TemplateOrSpecialization.dyn_cast<
                                         FunctionTemplateSpecializationInfo*>())
     return FTSInfo->getPointOfInstantiation();
-  else if (MemberSpecializationInfo *MSInfo
-             = TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo*>())
+  if (MemberSpecializationInfo *MSInfo =
+          TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo *>())
     return MSInfo->getPointOfInstantiation();
 
   return SourceLocation();
@@ -4057,29 +4052,29 @@ unsigned FunctionDecl::getMemoryFunctionKind() const {
     if (isExternC()) {
       if (FnInfo->isStr("memset"))
         return Builtin::BImemset;
-      else if (FnInfo->isStr("memcpy"))
+      if (FnInfo->isStr("memcpy"))
         return Builtin::BImemcpy;
-      else if (FnInfo->isStr("mempcpy"))
+      if (FnInfo->isStr("mempcpy"))
         return Builtin::BImempcpy;
-      else if (FnInfo->isStr("memmove"))
+      if (FnInfo->isStr("memmove"))
         return Builtin::BImemmove;
-      else if (FnInfo->isStr("memcmp"))
+      if (FnInfo->isStr("memcmp"))
         return Builtin::BImemcmp;
-      else if (FnInfo->isStr("bcmp"))
+      if (FnInfo->isStr("bcmp"))
         return Builtin::BIbcmp;
-      else if (FnInfo->isStr("strncpy"))
+      if (FnInfo->isStr("strncpy"))
         return Builtin::BIstrncpy;
-      else if (FnInfo->isStr("strncmp"))
+      if (FnInfo->isStr("strncmp"))
         return Builtin::BIstrncmp;
-      else if (FnInfo->isStr("strncasecmp"))
+      if (FnInfo->isStr("strncasecmp"))
         return Builtin::BIstrncasecmp;
-      else if (FnInfo->isStr("strncat"))
+      if (FnInfo->isStr("strncat"))
         return Builtin::BIstrncat;
-      else if (FnInfo->isStr("strndup"))
+      if (FnInfo->isStr("strndup"))
         return Builtin::BIstrndup;
-      else if (FnInfo->isStr("strlen"))
+      if (FnInfo->isStr("strlen"))
         return Builtin::BIstrlen;
-      else if (FnInfo->isStr("bzero"))
+      if (FnInfo->isStr("bzero"))
         return Builtin::BIbzero;
     } else if (isInStdNamespace()) {
       if (FnInfo->isStr("free"))
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index dc59f3dd1f15..c26d6d1a42ea 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -230,11 +230,11 @@ bool Decl::isTemplateDecl() const {
 TemplateDecl *Decl::getDescribedTemplate() const {
   if (auto *FD = dyn_cast<FunctionDecl>(this))
     return FD->getDescribedFunctionTemplate();
-  else if (auto *RD = dyn_cast<CXXRecordDecl>(this))
+  if (auto *RD = dyn_cast<CXXRecordDecl>(this))
     return RD->getDescribedClassTemplate();
-  else if (auto *VD = dyn_cast<VarDecl>(this))
+  if (auto *VD = dyn_cast<VarDecl>(this))
     return VD->getDescribedVarTemplate();
-  else if (auto *AD = dyn_cast<TypeAliasDecl>(this))
+  if (auto *AD = dyn_cast<TypeAliasDecl>(this))
     return AD->getDescribedAliasTemplate();
 
   return nullptr;
@@ -695,24 +695,23 @@ bool Decl::canBeWeakImported(bool &IsDefinition) const {
       return false;
     }
     return true;
-
+  }
   // Functions, if they aren't definitions.
-  } else if (const auto *FD = dyn_cast<FunctionDecl>(this)) {
+  if (const auto *FD = dyn_cast<FunctionDecl>(this)) {
     if (FD->hasBody()) {
       IsDefinition = true;
       return false;
     }
     return true;
 
+  }
   // Objective-C classes, if this is the non-fragile runtime.
-  } else if (isa<ObjCInterfaceDecl>(this) &&
+  if (isa<ObjCInterfaceDecl>(this) &&
              getASTContext().getLangOpts().ObjCRuntime.hasWeakClassImport()) {
     return true;
-
-  // Nothing else.
-  } else {
-    return false;
   }
+  // Nothing else.
+  return false;
 }
 
 bool Decl::isWeakImported() const {
@@ -1027,16 +1026,16 @@ template <class T> static Decl *getNonClosureContext(T *D) {
         MD->getParent()->isLambda())
       return getNonClosureContext(MD->getParent()->getParent());
     return MD;
-  } else if (auto *FD = dyn_cast<FunctionDecl>(D))
+  }
+  if (auto *FD = dyn_cast<FunctionDecl>(D))
     return FD;
-  else if (auto *MD = dyn_cast<ObjCMethodDecl>(D))
+  if (auto *MD = dyn_cast<ObjCMethodDecl>(D))
     return MD;
-  else if (auto *BD = dyn_cast<BlockDecl>(D))
+  if (auto *BD = dyn_cast<BlockDecl>(D))
     return getNonClosureContext(BD->getParent());
-  else if (auto *CD = dyn_cast<CapturedDecl>(D))
+  if (auto *CD = dyn_cast<CapturedDecl>(D))
     return getNonClosureContext(CD->getParent());
-  else
-    return nullptr;
+  return nullptr;
 }
 
 Decl *Decl::getNonClosureContext() {
@@ -1171,10 +1170,8 @@ bool DeclContext::isDependentContext() const {
 bool DeclContext::isTransparentContext() const {
   if (getDeclKind() == Decl::Enum)
     return !cast<EnumDecl>(this)->isScoped();
-  else if (getDeclKind() == Decl::LinkageSpec || getDeclKind() == Decl::Export)
-    return true;
 
-  return false;
+  return getDeclKind() == Decl::LinkageSpec || getDeclKind() == Decl::Export;
 }
 
 static bool isLinkageSpecContext(const DeclContext *DC,
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 00266eae42a4..2b82095f9862 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -117,7 +117,8 @@ const Expr *Expr::skipRValueSubobjectAdjustments(
           BO->getRHS()->getType()->getAs<MemberPointerType>();
         Adjustments.push_back(SubobjectAdjustment(MPT, BO->getRHS()));
         continue;
-      } else if (BO->getOpcode() == BO_Comma) {
+      }
+      if (BO->getOpcode() == BO_Comma) {
         CommaLHSs.push_back(BO->getLHS());
         E = BO->getRHS();
         continue;
@@ -659,8 +660,8 @@ std::string PredefinedExpr::ComputeName(IdentKind IK, const Decl *CurrentDecl) {
         if (!Buffer.empty() && Buffer.front() == '\01')
           return std::string(Buffer.substr(1));
         return std::string(Buffer.str());
-      } else
-        return std::string(ND->getIdentifier()->getName());
+      }
+      return std::string(ND->getIdentifier()->getName());
     }
     return "";
   }
@@ -3773,7 +3774,7 @@ Expr::isNullPointerConstant(ASTContext &Ctx,
     const IntegerLiteral *Lit = dyn_cast<IntegerLiteral>(this);
     if (Lit && !Lit->getValue())
       return NPCK_ZeroLiteral;
-    else if (!Ctx.getLangOpts().MSVCCompat || !isCXX98IntegralConstantExpr(Ctx))
+    if (!Ctx.getLangOpts().MSVCCompat || !isCXX98IntegralConstantExpr(Ctx))
       return NPCK_NotNull;
   } else {
     // If we have an integer constant expression, we need to *evaluate* it and
@@ -4202,9 +4203,8 @@ GenericSelectionExpr::CreateEmpty(const ASTContext &Context,
 IdentifierInfo *DesignatedInitExpr::Designator::getFieldName() const {
   assert(Kind == FieldDesignator && "Only valid on a field designator");
   if (Field.NameOrField & 0x01)
-    return reinterpret_cast<IdentifierInfo *>(Field.NameOrField&~0x01);
-  else
-    return getField()->getIdentifier();
+    return reinterpret_cast<IdentifierInfo *>(Field.NameOrField & ~0x01);
+  return getField()->getIdentifier();
 }
 
 DesignatedInitExpr::DesignatedInitExpr(const ASTContext &C, QualType Ty,
@@ -4322,7 +4322,8 @@ void DesignatedInitExpr::ExpandDesignator(const ASTContext &C, unsigned Idx,
                        Designators + Idx);
     --NumNewDesignators;
     return;
-  } else if (NumNewDesignators == 1) {
+  }
+  if (NumNewDesignators == 1) {
     Designators[Idx] = *First;
     return;
   }
diff --git a/clang/lib/Basic/CMakeLists.txt b/clang/lib/Basic/CMakeLists.txt
index 0ca4148801fc..b1c5003e172a 100644
--- a/clang/lib/Basic/CMakeLists.txt
+++ b/clang/lib/Basic/CMakeLists.txt
@@ -56,6 +56,7 @@ add_clang_library(clangBasic
   LangStandards.cpp
   Module.cpp
   ObjCRuntime.cpp
+  OpenCLOptions.cpp
   OpenMPKinds.cpp
   OperatorPrecedence.cpp
   SanitizerBlacklist.cpp
diff --git a/clang/lib/Basic/OpenCLOptions.cpp b/clang/lib/Basic/OpenCLOptions.cpp
new file mode 100644
index 000000000000..266acc5fe477
--- /dev/null
+++ b/clang/lib/Basic/OpenCLOptions.cpp
@@ -0,0 +1,106 @@
+//===--- OpenCLOptions.cpp---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/OpenCLOptions.h"
+
+namespace clang {
+
+bool OpenCLOptions::isKnown(llvm::StringRef Ext) const {
+  return OptMap.find(Ext) != OptMap.end();
+}
+
+bool OpenCLOptions::isEnabled(llvm::StringRef Ext) const {
+  auto E = OptMap.find(Ext);
+  return E != OptMap.end() && E->second.Enabled;
+}
+
+bool OpenCLOptions::isSupported(llvm::StringRef Ext,
+                                const LangOptions &LO) const {
+  auto E = OptMap.find(Ext);
+  if (E == OptMap.end()) {
+    return false;
+  }
+  auto I = OptMap.find(Ext)->getValue();
+  return I.Supported && I.isAvailableIn(LO);
+}
+
+bool OpenCLOptions::isSupportedCore(llvm::StringRef Ext,
+                                    const LangOptions &LO) const {
+  auto E = OptMap.find(Ext);
+  if (E == OptMap.end()) {
+    return false;
+  }
+  auto I = OptMap.find(Ext)->getValue();
+  return I.Supported && I.isCoreIn(LO);
+}
+
+bool OpenCLOptions::isSupportedOptionalCore(llvm::StringRef Ext,
+                                            const LangOptions &LO) const {
+  auto E = OptMap.find(Ext);
+  if (E == OptMap.end()) {
+    return false;
+  }
+  auto I = OptMap.find(Ext)->getValue();
+  return I.Supported && I.isOptionalCoreIn(LO);
+}
+
+bool OpenCLOptions::isSupportedCoreOrOptionalCore(llvm::StringRef Ext,
+                                                  const LangOptions &LO) const {
+  return isSupportedCore(Ext, LO) || isSupportedOptionalCore(Ext, LO);
+}
+
+bool OpenCLOptions::isSupportedExtension(llvm::StringRef Ext,
+                                         const LangOptions &LO) const {
+  auto E = OptMap.find(Ext);
+  if (E == OptMap.end()) {
+    return false;
+  }
+  auto I = OptMap.find(Ext)->getValue();
+  return I.Supported && I.isAvailableIn(LO) &&
+         !isSupportedCoreOrOptionalCore(Ext, LO);
+}
+
+void OpenCLOptions::enable(llvm::StringRef Ext, bool V) {
+  OptMap[Ext].Enabled = V;
+}
+
+void OpenCLOptions::support(llvm::StringRef Ext, bool V) {
+  assert(!Ext.empty() && "Extension is empty.");
+  assert(Ext[0] != '+' && Ext[0] != '-');
+  OptMap[Ext].Supported = V;
+}
+
+OpenCLOptions::OpenCLOptions() {
+#define OPENCL_GENERIC_EXTENSION(Ext, AvailVer, CoreVer, OptVer)               \
+  OptMap[#Ext].Avail = AvailVer;                                               \
+  OptMap[#Ext].Core = CoreVer;                                                 \
+  OptMap[#Ext].Opt = OptVer;
+#include "clang/Basic/OpenCLExtensions.def"
+}
+
+void OpenCLOptions::addSupport(const llvm::StringMap<bool> &FeaturesMap,
+                               const LangOptions &Opts) {
+  for (const auto &F : FeaturesMap) {
+    const auto &Name = F.getKey();
+    if (F.getValue() && isKnown(Name) && OptMap[Name].isAvailableIn(Opts))
+      support(Name);
+  }
+}
+
+void OpenCLOptions::disableAll() {
+  for (auto &Opt : OptMap)
+    Opt.getValue().Enabled = false;
+}
+
+void OpenCLOptions::enableSupportedCore(const LangOptions &LO) {
+  for (auto &Opt : OptMap)
+    if (isSupportedCoreOrOptionalCore(Opt.getKey(), LO))
+      Opt.getValue().Enabled = true;
+}
+
+} // end namespace clang
diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index e2cd44d5fb9b..4b2f9e7d7fb8 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -735,7 +735,7 @@ TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
     return nullptr;
 
   Target->setSupportedOpenCLOpts();
-  Target->setOpenCLExtensionOpts();
+  Target->setCommandLineOpenCLOpts();
   Target->setMaxAtomicWidth();
 
   if (!Target->validateTarget(Diags))
@@ -745,3 +745,29 @@ TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
 
   return Target.release();
 }
+
+/// getOpenCLFeatureDefines - Define OpenCL macros based on target settings
+/// and language version
+void TargetInfo::getOpenCLFeatureDefines(const LangOptions &Opts,
+                                         MacroBuilder &Builder) const {
+
+  auto defineOpenCLExtMacro = [&](llvm::StringRef Name, unsigned AvailVer,
+                                  unsigned CoreVersions,
+                                  unsigned OptionalVersions) {
+    // Check if extension is supported by target and is available in this
+    // OpenCL version
+    auto It = getTargetOpts().OpenCLFeaturesMap.find(Name);
+    if ((It != getTargetOpts().OpenCLFeaturesMap.end()) && It->getValue() &&
+        OpenCLOptions::OpenCLOptionInfo(AvailVer, CoreVersions,
+                                        OptionalVersions)
+            .isAvailableIn(Opts))
+      Builder.defineMacro(Name);
+  };
+#define OPENCL_GENERIC_EXTENSION(Ext, Avail, Core, Opt)                        \
+  defineOpenCLExtMacro(#Ext, Avail, Core, Opt);
+#include "clang/Basic/OpenCLExtensions.def"
+
+  // FIXME: OpenCL options which affect language semantics/syntax
+  // should be moved into LangOptions, thus macro definitions of
+  // such options is better to be done in clang::InitializePreprocessor
+}
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index fba1e4288ed1..8ee0ca30d305 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -284,33 +284,32 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
 
   void setSupportedOpenCLOpts() override {
     auto &Opts = getSupportedOpenCLOpts();
-    Opts.support("cl_clang_storage_class_specifiers");
-    Opts.support("__cl_clang_function_pointers");
-    Opts.support("__cl_clang_variadic_functions");
+    Opts["cl_clang_storage_class_specifiers"] = true;
+    Opts["__cl_clang_variadic_functions"] = true;
+    Opts["__cl_clang_function_pointers"] = true;
 
     bool IsAMDGCN = isAMDGCN(getTriple());
 
-    if (hasFP64())
-      Opts.support("cl_khr_fp64");
+    Opts["cl_khr_fp64"] = hasFP64();
 
     if (IsAMDGCN || GPUKind >= llvm::AMDGPU::GK_CEDAR) {
-      Opts.support("cl_khr_byte_addressable_store");
-      Opts.support("cl_khr_global_int32_base_atomics");
-      Opts.support("cl_khr_global_int32_extended_atomics");
-      Opts.support("cl_khr_local_int32_base_atomics");
-      Opts.support("cl_khr_local_int32_extended_atomics");
+      Opts["cl_khr_byte_addressable_store"] = true;
+      Opts["cl_khr_global_int32_base_atomics"] = true;
+      Opts["cl_khr_global_int32_extended_atomics"] = true;
+      Opts["cl_khr_local_int32_base_atomics"] = true;
+      Opts["cl_khr_local_int32_extended_atomics"] = true;
     }
 
     if (IsAMDGCN) {
-      Opts.support("cl_khr_fp16");
-      Opts.support("cl_khr_int64_base_atomics");
-      Opts.support("cl_khr_int64_extended_atomics");
-      Opts.support("cl_khr_mipmap_image");
-      Opts.support("cl_khr_mipmap_image_writes");
-      Opts.support("cl_khr_subgroups");
-      Opts.support("cl_khr_3d_image_writes");
-      Opts.support("cl_amd_media_ops");
-      Opts.support("cl_amd_media_ops2");
+      Opts["cl_khr_fp16"] = true;
+      Opts["cl_khr_int64_base_atomics"] = true;
+      Opts["cl_khr_int64_extended_atomics"] = true;
+      Opts["cl_khr_mipmap_image"] = true;
+      Opts["cl_khr_mipmap_image_writes"] = true;
+      Opts["cl_khr_subgroups"] = true;
+      Opts["cl_khr_3d_image_writes"] = true;
+      Opts["cl_amd_media_ops"] = true;
+      Opts["cl_amd_media_ops2"] = true;
     }
   }
 
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
index bf39592815e7..2b4f0ec3ecca 100644
--- a/clang/lib/Basic/Targets/NVPTX.h
+++ b/clang/lib/Basic/Targets/NVPTX.h
@@ -127,24 +127,24 @@ class LLVM_LIBRARY_VISIBILITY NVPTXTargetInfo : public TargetInfo {
 
   void setSupportedOpenCLOpts() override {
     auto &Opts = getSupportedOpenCLOpts();
-    Opts.support("cl_clang_storage_class_specifiers");
-    Opts.support("__cl_clang_function_pointers");
-    Opts.support("__cl_clang_variadic_functions");
-
-    Opts.support("cl_khr_fp64");
-    Opts.support("cl_khr_byte_addressable_store");
-    Opts.support("cl_khr_global_int32_base_atomics");
-    Opts.support("cl_khr_global_int32_extended_atomics");
-    Opts.support("cl_khr_local_int32_base_atomics");
-    Opts.support("cl_khr_local_int32_extended_atomics");
+    Opts["cl_clang_storage_class_specifiers"] = true;
+    Opts["__cl_clang_function_pointers"] = true;
+    Opts["__cl_clang_variadic_functions"] = true;
+
+    Opts["cl_khr_fp64"] = true;
+    Opts["cl_khr_byte_addressable_store"] = true;
+    Opts["cl_khr_global_int32_base_atomics"] = true;
+    Opts["cl_khr_global_int32_extended_atomics"] = true;
+    Opts["cl_khr_local_int32_base_atomics"] = true;
+    Opts["cl_khr_local_int32_extended_atomics"] = true;
     // PTX actually supports 64 bits operations even if the Nvidia OpenCL
     // runtime does not report support for it.
     // This is required for libclc to compile 64 bits atomic functions.
     // FIXME: maybe we should have a way to control this ?
-    Opts.support("cl_khr_int64_base_atomics");
-    Opts.support("cl_khr_int64_extended_atomics");
-    Opts.support("cl_khr_fp16");
-    Opts.support("cl_khr_3d_image_writes");
+    Opts["cl_khr_int64_base_atomics"] = true;
+    Opts["cl_khr_int64_extended_atomics"] = true;
+    Opts["cl_khr_fp16"] = true;
+    Opts["cl_khr_3d_image_writes"] = true;
   }
 
   /// \returns If a target requires an address within a target specific address
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 4436db904d59..7e6118af38e0 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -109,13 +109,18 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   if (ABIName == "ilp32e")
     Builder.defineMacro("__riscv_abi_rve");
 
+  Builder.defineMacro("__riscv_arch_test");
+  Builder.defineMacro("__riscv_i", "2000000");
+
   if (HasM) {
+    Builder.defineMacro("__riscv_m", "2000000");
     Builder.defineMacro("__riscv_mul");
     Builder.defineMacro("__riscv_div");
     Builder.defineMacro("__riscv_muldiv");
   }
 
   if (HasA) {
+    Builder.defineMacro("__riscv_a", "2000000");
     Builder.defineMacro("__riscv_atomic");
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
@@ -125,22 +130,71 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   }
 
   if (HasF || HasD) {
+    Builder.defineMacro("__riscv_f", "2000000");
     Builder.defineMacro("__riscv_flen", HasD ? "64" : "32");
     Builder.defineMacro("__riscv_fdiv");
     Builder.defineMacro("__riscv_fsqrt");
   }
 
-  if (HasC)
+  if (HasD)
+    Builder.defineMacro("__riscv_d", "2000000");
+
+  if (HasC) {
+    Builder.defineMacro("__riscv_c", "2000000");
     Builder.defineMacro("__riscv_compressed");
+  }
 
-  if (HasB)
+  if (HasB) {
+    Builder.defineMacro("__riscv_b", "93000");
     Builder.defineMacro("__riscv_bitmanip");
+  }
 
-  if (HasV)
+  if (HasV) {
+    Builder.defineMacro("__riscv_v", "9000");
     Builder.defineMacro("__riscv_vector");
+  }
+
+  if (HasZba)
+    Builder.defineMacro("__riscv_zba", "93000");
+
+  if (HasZbb)
+    Builder.defineMacro("__riscv_zbb", "93000");
+
+  if (HasZbc)
+    Builder.defineMacro("__riscv_zbc", "93000");
+
+  if (HasZbe)
+    Builder.defineMacro("__riscv_zbe", "93000");
+
+  if (HasZbf)
+    Builder.defineMacro("__riscv_zbf", "93000");
+
+  if (HasZbm)
+    Builder.defineMacro("__riscv_zbm", "93000");
+
+  if (HasZbp)
+    Builder.defineMacro("__riscv_zbp", "93000");
+
+  if (HasZbproposedc)
+    Builder.defineMacro("__riscv_zbproposedc", "93000");
+
+  if (HasZbr)
+    Builder.defineMacro("__riscv_zbr", "93000");
+
+  if (HasZbs)
+    Builder.defineMacro("__riscv_zbs", "93000");
+
+  if (HasZbt)
+    Builder.defineMacro("__riscv_zbt", "93000");
 
   if (HasZfh)
-    Builder.defineMacro("__riscv_zfh");
+    Builder.defineMacro("__riscv_zfh", "1000");
+
+  if (HasZvamo)
+    Builder.defineMacro("__riscv_zvamo", "9000");
+
+  if (HasZvlsseg)
+    Builder.defineMacro("__riscv_zvlsseg", "9000");
 }
 
 /// Return true if has this feature, need to sync with handleTargetFeatures.
@@ -157,7 +211,20 @@ bool RISCVTargetInfo::hasFeature(StringRef Feature) const {
       .Case("c", HasC)
       .Case("experimental-b", HasB)
       .Case("experimental-v", HasV)
+      .Case("experimental-zba", HasZba)
+      .Case("experimental-zbb", HasZbb)
+      .Case("experimental-zbc", HasZbc)
+      .Case("experimental-zbe", HasZbe)
+      .Case("experimental-zbf", HasZbf)
+      .Case("experimental-zbm", HasZbm)
+      .Case("experimental-zbp", HasZbp)
+      .Case("experimental-zbproposedc", HasZbproposedc)
+      .Case("experimental-zbr", HasZbr)
+      .Case("experimental-zbs", HasZbs)
+      .Case("experimental-zbt", HasZbt)
       .Case("experimental-zfh", HasZfh)
+      .Case("experimental-zvamo", HasZvamo)
+      .Case("experimental-zvlsseg", HasZvlsseg)
       .Default(false);
 }
 
@@ -179,8 +246,34 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasB = true;
     else if (Feature == "+experimental-v")
       HasV = true;
+    else if (Feature == "+experimental-zba")
+      HasZba = true;
+    else if (Feature == "+experimental-zbb")
+      HasZbb = true;
+    else if (Feature == "+experimental-zbc")
+      HasZbc = true;
+    else if (Feature == "+experimental-zbe")
+      HasZbe = true;
+    else if (Feature == "+experimental-zbf")
+      HasZbf = true;
+    else if (Feature == "+experimental-zbm")
+      HasZbm = true;
+    else if (Feature == "+experimental-zbp")
+      HasZbp = true;
+    else if (Feature == "+experimental-zbproposedc")
+      HasZbproposedc = true;
+    else if (Feature == "+experimental-zbr")
+      HasZbr = true;
+    else if (Feature == "+experimental-zbs")
+      HasZbs = true;
+    else if (Feature == "+experimental-zbt")
+      HasZbt = true;
     else if (Feature == "+experimental-zfh")
       HasZfh = true;
+    else if (Feature == "+experimental-zvamo")
+      HasZvamo = true;
+    else if (Feature == "+experimental-zvlsseg")
+      HasZvlsseg = true;
   }
 
   return true;
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index 8407798eb149..f1e9215b2d17 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -32,7 +32,20 @@ class RISCVTargetInfo : public TargetInfo {
   bool HasC = false;
   bool HasB = false;
   bool HasV = false;
+  bool HasZba = false;
+  bool HasZbb = false;
+  bool HasZbc = false;
+  bool HasZbe = false;
+  bool HasZbf = false;
+  bool HasZbm = false;
+  bool HasZbp = false;
+  bool HasZbproposedc = false;
+  bool HasZbr = false;
+  bool HasZbs = false;
+  bool HasZbt = false;
   bool HasZfh = false;
+  bool HasZvamo = false;
+  bool HasZvlsseg = false;
 
 public:
   RISCVTargetInfo(const llvm::Triple &Triple, const TargetOptions &)
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 3b9000133438..9b0250f658bb 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -117,7 +117,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRTargetInfo : public TargetInfo {
   void setSupportedOpenCLOpts() override {
     // Assume all OpenCL extensions and optional core features are supported
     // for SPIR since it is a generic target.
-    getSupportedOpenCLOpts().supportAll();
+    supportAllOpenCLOpts();
   }
 
   bool hasExtIntType() const override { return true; }
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 9979460a55bf..91a365c7d405 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -359,9 +359,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
 
   bool hasSjLjLowering() const override { return true; }
 
-  void setSupportedOpenCLOpts() override {
-    getSupportedOpenCLOpts().supportAll();
-  }
+  void setSupportedOpenCLOpts() override { supportAllOpenCLOpts(); }
 
   uint64_t getPointerWidthV(unsigned AddrSpace) const override {
     if (AddrSpace == ptr32_sptr || AddrSpace == ptr32_uptr)
diff --git a/clang/lib/Driver/ToolChains/AVR.cpp b/clang/lib/Driver/ToolChains/AVR.cpp
index c999c3b87f84..ae56b7b5249e 100644
--- a/clang/lib/Driver/ToolChains/AVR.cpp
+++ b/clang/lib/Driver/ToolChains/AVR.cpp
@@ -32,248 +32,247 @@ const struct {
   StringRef Name;
   std::string SubPath;
   StringRef Family;
-  unsigned DataAddr;
 } MCUInfo[] = {
-    {"at90s1200", "", "avr1", 0},
-    {"attiny11", "", "avr1", 0},
-    {"attiny12", "", "avr1", 0},
-    {"attiny15", "", "avr1", 0},
-    {"attiny28", "", "avr1", 0},
-    {"at90s2313", "tiny-stack", "avr2", 0x800060},
-    {"at90s2323", "tiny-stack", "avr2", 0x800060},
-    {"at90s2333", "tiny-stack", "avr2", 0x800060},
-    {"at90s2343", "tiny-stack", "avr2", 0x800060},
-    {"at90s4433", "tiny-stack", "avr2, 0x800060"},
-    {"attiny22", "tiny-stack", "avr2", 0x800060},
-    {"attiny26", "tiny-stack", "avr2", 0x800060},
-    {"at90s4414", "", "avr2", 0x800060},
-    {"at90s4434", "", "avr2", 0x800060},
-    {"at90s8515", "", "avr2", 0x800060},
-    {"at90c8534", "", "avr2", 0x800060},
-    {"at90s8535", "", "avr2", 0x800060},
-    {"attiny13", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny13a", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny2313", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny2313a", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny24", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny24a", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny25", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny261", "avr25/tiny-stack", "avr25", 0x800060},
-    {"attiny261a", "avr25/tiny-stack", "avr25", 0x800060},
-    {"at86rf401", "avr25", "avr25", 0x800060},
-    {"ata5272", "avr25", "avr25", 0x800100},
-    {"attiny4313", "avr25", "avr25", 0x800060},
-    {"attiny44", "avr25", "avr25", 0x800060},
-    {"attiny44a", "avr25", "avr25", 0x800060},
-    {"attiny84", "avr25", "avr25", 0x800060},
-    {"attiny84a", "avr25", "avr25", 0x800060},
-    {"attiny45", "avr25", "avr25", 0x800060},
-    {"attiny85", "avr25", "avr25", 0x800060},
-    {"attiny441", "avr25", "avr25", 0x800100},
-    {"attiny461", "avr25", "avr25", 0x800060},
-    {"attiny461a", "avr25", "avr25", 0x800060},
-    {"attiny841", "avr25", "avr25", 0x800100},
-    {"attiny861", "avr25", "avr25", 0x800060},
-    {"attiny861a", "avr25", "avr25", 0x800060},
-    {"attiny87", "avr25", "avr25", 0x800100},
-    {"attiny43u", "avr25", "avr25", 0x800060},
-    {"attiny48", "avr25", "avr25", 0x800100},
-    {"attiny88", "avr25", "avr25", 0x800100},
-    {"attiny828", "avr25", "avr25", 0x800100},
-    {"at43usb355", "avr3", "avr3", 0x800100},
-    {"at76c711", "avr3", "avr3", 0x800060},
-    {"atmega103", "avr31", "avr31", 0x800060},
-    {"at43usb320", "avr31", "avr31", 0x800060},
-    {"attiny167", "avr35", "avr35", 0x800100},
-    {"at90usb82", "avr35", "avr35", 0x800100},
-    {"at90usb162", "avr35", "avr35", 0x800100},
-    {"ata5505", "avr35", "avr35", 0x800100},
-    {"atmega8u2", "avr35", "avr35", 0x800100},
-    {"atmega16u2", "avr35", "avr35", 0x800100},
-    {"atmega32u2", "avr35", "avr35", 0x800100},
-    {"attiny1634", "avr35", "avr35", 0x800100},
-    {"atmega8", "avr4", "avr4", 0x800060},
-    {"ata6289", "avr4", "avr4", 0x800100},
-    {"atmega8a", "avr4", "avr4", 0x800060},
-    {"ata6285", "avr4", "avr4", 0x800100},
-    {"ata6286", "avr4", "avr4", 0x800100},
-    {"atmega48", "avr4", "avr4", 0x800100},
-    {"atmega48a", "avr4", "avr4", 0x800100},
-    {"atmega48pa", "avr4", "avr4", 0x800100},
-    {"atmega48pb", "avr4", "avr4", 0x800100},
-    {"atmega48p", "avr4", "avr4", 0x800100},
-    {"atmega88", "avr4", "avr4", 0x800100},
-    {"atmega88a", "avr4", "avr4", 0x800100},
-    {"atmega88p", "avr4", "avr4", 0x800100},
-    {"atmega88pa", "avr4", "avr4", 0x800100},
-    {"atmega88pb", "avr4", "avr4", 0x800100},
-    {"atmega8515", "avr4", "avr4", 0x800060},
-    {"atmega8535", "avr4", "avr4", 0x800060},
-    {"atmega8hva", "avr4", "avr4", 0x800100},
-    {"at90pwm1", "avr4", "avr4", 0x800100},
-    {"at90pwm2", "avr4", "avr4", 0x800100},
-    {"at90pwm2b", "avr4", "avr4", 0x800100},
-    {"at90pwm3", "avr4", "avr4", 0x800100},
-    {"at90pwm3b", "avr4", "avr4", 0x800100},
-    {"at90pwm81", "avr4", "avr4", 0x800100},
-    {"ata5790", "avr5", "avr5", 0x800100},
-    {"ata5795", "avr5", "avr5", 0x800100},
-    {"atmega16", "avr5", "avr5", 0x800060},
-    {"atmega16a", "avr5", "avr5", 0x800060},
-    {"atmega161", "avr5", "avr5", 0x800060},
-    {"atmega162", "avr5", "avr5", 0x800100},
-    {"atmega163", "avr5", "avr5", 0x800060},
-    {"atmega164a", "avr5", "avr5", 0x800100},
-    {"atmega164p", "avr5", "avr5", 0x800100},
-    {"atmega164pa", "avr5", "avr5", 0x800100},
-    {"atmega165", "avr5", "avr5", 0x800100},
-    {"atmega165a", "avr5", "avr5", 0x800100},
-    {"atmega165p", "avr5", "avr5", 0x800100},
-    {"atmega165pa", "avr5", "avr5", 0x800100},
-    {"atmega168", "avr5", "avr5", 0x800100},
-    {"atmega168a", "avr5", "avr5", 0x800100},
-    {"atmega168p", "avr5", "avr5", 0x800100},
-    {"atmega168pa", "avr5", "avr5", 0x800100},
-    {"atmega168pb", "avr5", "avr5", 0x800100},
-    {"atmega169", "avr5", "avr5", 0x800100},
-    {"atmega169a", "avr5", "avr5", 0x800100},
-    {"atmega169p", "avr5", "avr5", 0x800100},
-    {"atmega169pa", "avr5", "avr5", 0x800100},
-    {"atmega32", "avr5", "avr5", 0x800060},
-    {"atmega32a", "avr5", "avr5", 0x800060},
-    {"atmega323", "avr5", "avr5", 0x800060},
-    {"atmega324a", "avr5", "avr5", 0x800100},
-    {"atmega324p", "avr5", "avr5", 0x800100},
-    {"atmega324pa", "avr5", "avr5", 0x800100},
-    {"atmega325", "avr5", "avr5", 0x800100},
-    {"atmega325a", "avr5", "avr5", 0x800100},
-    {"atmega325p", "avr5", "avr5", 0x800100},
-    {"atmega325pa", "avr5", "avr5", 0x800100},
-    {"atmega3250", "avr5", "avr5", 0x800100},
-    {"atmega3250a", "avr5", "avr5", 0x800100},
-    {"atmega3250p", "avr5", "avr5", 0x800100},
-    {"atmega3250pa", "avr5", "avr5", 0x800100},
-    {"atmega328", "avr5", "avr5", 0x800100},
-    {"atmega328p", "avr5", "avr5", 0x800100},
-    {"atmega329", "avr5", "avr5", 0x800100},
-    {"atmega329a", "avr5", "avr5", 0x800100},
-    {"atmega329p", "avr5", "avr5", 0x800100},
-    {"atmega329pa", "avr5", "avr5", 0x800100},
-    {"atmega3290", "avr5", "avr5", 0x800100},
-    {"atmega3290a", "avr5", "avr5", 0x800100},
-    {"atmega3290p", "avr5", "avr5", 0x800100},
-    {"atmega3290pa", "avr5", "avr5", 0x800100},
-    {"atmega406", "avr5", "avr5", 0x800100},
-    {"atmega64", "avr5", "avr5", 0x800100},
-    {"atmega64a", "avr5", "avr5", 0x800100},
-    {"atmega640", "avr5", "avr5", 0x800200},
-    {"atmega644", "avr5", "avr5", 0x800100},
-    {"atmega644a", "avr5", "avr5", 0x800100},
-    {"atmega644p", "avr5", "avr5", 0x800100},
-    {"atmega644pa", "avr5", "avr5", 0x800100},
-    {"atmega645", "avr5", "avr5", 0x800100},
-    {"atmega645a", "avr5", "avr5", 0x800100},
-    {"atmega645p", "avr5", "avr5", 0x800100},
-    {"atmega649", "avr5", "avr5", 0x800100},
-    {"atmega649a", "avr5", "avr5", 0x800100},
-    {"atmega649p", "avr5", "avr5", 0x800100},
-    {"atmega6450", "avr5", "avr5", 0x800100},
-    {"atmega6450a", "avr5", "avr5", 0x800100},
-    {"atmega6450p", "avr5", "avr5", 0x800100},
-    {"atmega6490", "avr5", "avr5", 0x800100},
-    {"atmega6490a", "avr5", "avr5", 0x800100},
-    {"atmega6490p", "avr5", "avr5", 0x800100},
-    {"atmega64rfr2", "avr5", "avr5", 0x800200},
-    {"atmega644rfr2", "avr5", "avr5", 0x800200},
-    {"atmega16hva", "avr5", "avr5", 0x800100},
-    {"atmega16hva2", "avr5", "avr5", 0x800100},
-    {"atmega16hvb", "avr5", "avr5", 0x800100},
-    {"atmega16hvbrevb", "avr5", "avr5", 0x800100},
-    {"atmega32hvb", "avr5", "avr5", 0x800100},
-    {"atmega32hvbrevb", "avr5", "avr5", 0x800100},
-    {"atmega64hve", "avr5", "avr5", 0x800100},
-    {"at90can32", "avr5", "avr5", 0x800100},
-    {"at90can64", "avr5", "avr5", 0x800100},
-    {"at90pwm161", "avr5", "avr5", 0x800100},
-    {"at90pwm216", "avr5", "avr5", 0x800100},
-    {"at90pwm316", "avr5", "avr5", 0x800100},
-    {"atmega32c1", "avr5", "avr5", 0x800100},
-    {"atmega64c1", "avr5", "avr5", 0x800100},
-    {"atmega16m1", "avr5", "avr5", 0x800100},
-    {"atmega32m1", "avr5", "avr5", 0x800100},
-    {"atmega64m1", "avr5", "avr5", 0x800100},
-    {"atmega16u4", "avr5", "avr5", 0x800100},
-    {"atmega32u4", "avr5", "avr5", 0x800100},
-    {"atmega32u6", "avr5", "avr5", 0x800100},
-    {"at90usb646", "avr5", "avr5", 0x800100},
-    {"at90usb647", "avr5", "avr5", 0x800100},
-    {"at90scr100", "avr5", "avr5", 0x800100},
-    {"at94k", "avr5", "avr5", 0x800060},
-    {"m3000", "avr5", "avr5", 0x800060},
-    {"atmega128", "avr51", "avr51", 0x800100},
-    {"atmega128a", "avr51", "avr51", 0x800100},
-    {"atmega1280", "avr51", "avr51", 0x800200},
-    {"atmega1281", "avr51", "avr51", 0x800200},
-    {"atmega1284", "avr51", "avr51", 0x800100},
-    {"atmega1284p", "avr51", "avr51", 0x800100},
-    {"atmega128rfa1", "avr51", "avr51", 0x800200},
-    {"atmega128rfr2", "avr51", "avr51", 0x800200},
-    {"atmega1284rfr2", "avr51", "avr51", 0x800200},
-    {"at90can128", "avr51", "avr51", 0x800200},
-    {"at90usb1286", "avr51", "avr51", 0x800200},
-    {"at90usb1287", "avr51", "avr51", 0x800200},
-    {"atmega2560", "avr6", "avr6", 0x800200},
-    {"atmega2561", "avr6", "avr6", 0x800200},
-    {"atmega256rfr2", "avr6", "avr6", 0x800200},
-    {"atmega2564rfr2", "avr6", "avr6", 0x800200},
-    {"attiny4", "avrtiny", "avrtiny", 0x800040},
-    {"attiny5", "avrtiny", "avrtiny", 0x800040},
-    {"attiny9", "avrtiny", "avrtiny", 0x800040},
-    {"attiny10", "avrtiny", "avrtiny", 0x800040},
-    {"attiny20", "avrtiny", "avrtiny", 0x800040},
-    {"attiny40", "avrtiny", "avrtiny", 0x800040},
-    {"atxmega16a4", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega16a4u", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega16c4", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega16d4", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega32a4", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega32a4u", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega32c4", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega32d4", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega32e5", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega16e5", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega8e5", "avrxmega2", "avrxmega2", 0x802000},
-    {"atxmega64a3u", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64a4u", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64b1", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64b3", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64c3", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64d3", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64d4", "avrxmega4", "avrxmega4", 0x802000},
-    {"atxmega64a1", "avrxmega5", "avrxmega5", 0x802000},
-    {"atxmega64a1u", "avrxmega5", "avrxmega5", 0x802000},
-    {"atxmega128a3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128a3u", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128b1", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128b3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128c3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128d3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128d4", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega192a3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega192a3u", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega192c3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega192d3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega256a3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega256a3u", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega256a3b", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega256a3bu", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega256c3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega256d3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega384c3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega384d3", "avrxmega6", "avrxmega6", 0x802000},
-    {"atxmega128a1", "avrxmega7", "avrxmega7", 0x802000},
-    {"atxmega128a1u", "avrxmega7", "avrxmega7", 0x802000},
-    {"atxmega128a4u", "avrxmega7", "avrxmega7", 0x802000},
+    {"at90s1200", "", "avr1"},
+    {"attiny11", "", "avr1"},
+    {"attiny12", "", "avr1"},
+    {"attiny15", "", "avr1"},
+    {"attiny28", "", "avr1"},
+    {"at90s2313", "tiny-stack", "avr2"},
+    {"at90s2323", "tiny-stack", "avr2"},
+    {"at90s2333", "tiny-stack", "avr2"},
+    {"at90s2343", "tiny-stack", "avr2"},
+    {"at90s4433", "tiny-stack", "avr2"},
+    {"attiny22", "tiny-stack", "avr2"},
+    {"attiny26", "tiny-stack", "avr2"},
+    {"at90s4414", "", "avr2"},
+    {"at90s4434", "", "avr2"},
+    {"at90s8515", "", "avr2"},
+    {"at90c8534", "", "avr2"},
+    {"at90s8535", "", "avr2"},
+    {"attiny13", "avr25/tiny-stack", "avr25"},
+    {"attiny13a", "avr25/tiny-stack", "avr25"},
+    {"attiny2313", "avr25/tiny-stack", "avr25"},
+    {"attiny2313a", "avr25/tiny-stack", "avr25"},
+    {"attiny24", "avr25/tiny-stack", "avr25"},
+    {"attiny24a", "avr25/tiny-stack", "avr25"},
+    {"attiny25", "avr25/tiny-stack", "avr25"},
+    {"attiny261", "avr25/tiny-stack", "avr25"},
+    {"attiny261a", "avr25/tiny-stack", "avr25"},
+    {"at86rf401", "avr25", "avr25"},
+    {"ata5272", "avr25", "avr25"},
+    {"attiny4313", "avr25", "avr25"},
+    {"attiny44", "avr25", "avr25"},
+    {"attiny44a", "avr25", "avr25"},
+    {"attiny84", "avr25", "avr25"},
+    {"attiny84a", "avr25", "avr25"},
+    {"attiny45", "avr25", "avr25"},
+    {"attiny85", "avr25", "avr25"},
+    {"attiny441", "avr25", "avr25"},
+    {"attiny461", "avr25", "avr25"},
+    {"attiny461a", "avr25", "avr25"},
+    {"attiny841", "avr25", "avr25"},
+    {"attiny861", "avr25", "avr25"},
+    {"attiny861a", "avr25", "avr25"},
+    {"attiny87", "avr25", "avr25"},
+    {"attiny43u", "avr25", "avr25"},
+    {"attiny48", "avr25", "avr25"},
+    {"attiny88", "avr25", "avr25"},
+    {"attiny828", "avr25", "avr25"},
+    {"at43usb355", "avr3", "avr3"},
+    {"at76c711", "avr3", "avr3"},
+    {"atmega103", "avr31", "avr31"},
+    {"at43usb320", "avr31", "avr31"},
+    {"attiny167", "avr35", "avr35"},
+    {"at90usb82", "avr35", "avr35"},
+    {"at90usb162", "avr35", "avr35"},
+    {"ata5505", "avr35", "avr35"},
+    {"atmega8u2", "avr35", "avr35"},
+    {"atmega16u2", "avr35", "avr35"},
+    {"atmega32u2", "avr35", "avr35"},
+    {"attiny1634", "avr35", "avr35"},
+    {"atmega8", "avr4", "avr4"},
+    {"ata6289", "avr4", "avr4"},
+    {"atmega8a", "avr4", "avr4"},
+    {"ata6285", "avr4", "avr4"},
+    {"ata6286", "avr4", "avr4"},
+    {"atmega48", "avr4", "avr4"},
+    {"atmega48a", "avr4", "avr4"},
+    {"atmega48pa", "avr4", "avr4"},
+    {"atmega48pb", "avr4", "avr4"},
+    {"atmega48p", "avr4", "avr4"},
+    {"atmega88", "avr4", "avr4"},
+    {"atmega88a", "avr4", "avr4"},
+    {"atmega88p", "avr4", "avr4"},
+    {"atmega88pa", "avr4", "avr4"},
+    {"atmega88pb", "avr4", "avr4"},
+    {"atmega8515", "avr4", "avr4"},
+    {"atmega8535", "avr4", "avr4"},
+    {"atmega8hva", "avr4", "avr4"},
+    {"at90pwm1", "avr4", "avr4"},
+    {"at90pwm2", "avr4", "avr4"},
+    {"at90pwm2b", "avr4", "avr4"},
+    {"at90pwm3", "avr4", "avr4"},
+    {"at90pwm3b", "avr4", "avr4"},
+    {"at90pwm81", "avr4", "avr4"},
+    {"ata5790", "avr5", "avr5"},
+    {"ata5795", "avr5", "avr5"},
+    {"atmega16", "avr5", "avr5"},
+    {"atmega16a", "avr5", "avr5"},
+    {"atmega161", "avr5", "avr5"},
+    {"atmega162", "avr5", "avr5"},
+    {"atmega163", "avr5", "avr5"},
+    {"atmega164a", "avr5", "avr5"},
+    {"atmega164p", "avr5", "avr5"},
+    {"atmega164pa", "avr5", "avr5"},
+    {"atmega165", "avr5", "avr5"},
+    {"atmega165a", "avr5", "avr5"},
+    {"atmega165p", "avr5", "avr5"},
+    {"atmega165pa", "avr5", "avr5"},
+    {"atmega168", "avr5", "avr5"},
+    {"atmega168a", "avr5", "avr5"},
+    {"atmega168p", "avr5", "avr5"},
+    {"atmega168pa", "avr5", "avr5"},
+    {"atmega168pb", "avr5", "avr5"},
+    {"atmega169", "avr5", "avr5"},
+    {"atmega169a", "avr5", "avr5"},
+    {"atmega169p", "avr5", "avr5"},
+    {"atmega169pa", "avr5", "avr5"},
+    {"atmega32", "avr5", "avr5"},
+    {"atmega32a", "avr5", "avr5"},
+    {"atmega323", "avr5", "avr5"},
+    {"atmega324a", "avr5", "avr5"},
+    {"atmega324p", "avr5", "avr5"},
+    {"atmega324pa", "avr5", "avr5"},
+    {"atmega325", "avr5", "avr5"},
+    {"atmega325a", "avr5", "avr5"},
+    {"atmega325p", "avr5", "avr5"},
+    {"atmega325pa", "avr5", "avr5"},
+    {"atmega3250", "avr5", "avr5"},
+    {"atmega3250a", "avr5", "avr5"},
+    {"atmega3250p", "avr5", "avr5"},
+    {"atmega3250pa", "avr5", "avr5"},
+    {"atmega328", "avr5", "avr5"},
+    {"atmega328p", "avr5", "avr5"},
+    {"atmega329", "avr5", "avr5"},
+    {"atmega329a", "avr5", "avr5"},
+    {"atmega329p", "avr5", "avr5"},
+    {"atmega329pa", "avr5", "avr5"},
+    {"atmega3290", "avr5", "avr5"},
+    {"atmega3290a", "avr5", "avr5"},
+    {"atmega3290p", "avr5", "avr5"},
+    {"atmega3290pa", "avr5", "avr5"},
+    {"atmega406", "avr5", "avr5"},
+    {"atmega64", "avr5", "avr5"},
+    {"atmega64a", "avr5", "avr5"},
+    {"atmega640", "avr5", "avr5"},
+    {"atmega644", "avr5", "avr5"},
+    {"atmega644a", "avr5", "avr5"},
+    {"atmega644p", "avr5", "avr5"},
+    {"atmega644pa", "avr5", "avr5"},
+    {"atmega645", "avr5", "avr5"},
+    {"atmega645a", "avr5", "avr5"},
+    {"atmega645p", "avr5", "avr5"},
+    {"atmega649", "avr5", "avr5"},
+    {"atmega649a", "avr5", "avr5"},
+    {"atmega649p", "avr5", "avr5"},
+    {"atmega6450", "avr5", "avr5"},
+    {"atmega6450a", "avr5", "avr5"},
+    {"atmega6450p", "avr5", "avr5"},
+    {"atmega6490", "avr5", "avr5"},
+    {"atmega6490a", "avr5", "avr5"},
+    {"atmega6490p", "avr5", "avr5"},
+    {"atmega64rfr2", "avr5", "avr5"},
+    {"atmega644rfr2", "avr5", "avr5"},
+    {"atmega16hva", "avr5", "avr5"},
+    {"atmega16hva2", "avr5", "avr5"},
+    {"atmega16hvb", "avr5", "avr5"},
+    {"atmega16hvbrevb", "avr5", "avr5"},
+    {"atmega32hvb", "avr5", "avr5"},
+    {"atmega32hvbrevb", "avr5", "avr5"},
+    {"atmega64hve", "avr5", "avr5"},
+    {"at90can32", "avr5", "avr5"},
+    {"at90can64", "avr5", "avr5"},
+    {"at90pwm161", "avr5", "avr5"},
+    {"at90pwm216", "avr5", "avr5"},
+    {"at90pwm316", "avr5", "avr5"},
+    {"atmega32c1", "avr5", "avr5"},
+    {"atmega64c1", "avr5", "avr5"},
+    {"atmega16m1", "avr5", "avr5"},
+    {"atmega32m1", "avr5", "avr5"},
+    {"atmega64m1", "avr5", "avr5"},
+    {"atmega16u4", "avr5", "avr5"},
+    {"atmega32u4", "avr5", "avr5"},
+    {"atmega32u6", "avr5", "avr5"},
+    {"at90usb646", "avr5", "avr5"},
+    {"at90usb647", "avr5", "avr5"},
+    {"at90scr100", "avr5", "avr5"},
+    {"at94k", "avr5", "avr5"},
+    {"m3000", "avr5", "avr5"},
+    {"atmega128", "avr51", "avr51"},
+    {"atmega128a", "avr51", "avr51"},
+    {"atmega1280", "avr51", "avr51"},
+    {"atmega1281", "avr51", "avr51"},
+    {"atmega1284", "avr51", "avr51"},
+    {"atmega1284p", "avr51", "avr51"},
+    {"atmega128rfa1", "avr51", "avr51"},
+    {"atmega128rfr2", "avr51", "avr51"},
+    {"atmega1284rfr2", "avr51", "avr51"},
+    {"at90can128", "avr51", "avr51"},
+    {"at90usb1286", "avr51", "avr51"},
+    {"at90usb1287", "avr51", "avr51"},
+    {"atmega2560", "avr6", "avr6"},
+    {"atmega2561", "avr6", "avr6"},
+    {"atmega256rfr2", "avr6", "avr6"},
+    {"atmega2564rfr2", "avr6", "avr6"},
+    {"attiny4", "avrtiny", "avrtiny"},
+    {"attiny5", "avrtiny", "avrtiny"},
+    {"attiny9", "avrtiny", "avrtiny"},
+    {"attiny10", "avrtiny", "avrtiny"},
+    {"attiny20", "avrtiny", "avrtiny"},
+    {"attiny40", "avrtiny", "avrtiny"},
+    {"atxmega16a4", "avrxmega2", "avrxmega2"},
+    {"atxmega16a4u", "avrxmega2", "avrxmega2"},
+    {"atxmega16c4", "avrxmega2", "avrxmega2"},
+    {"atxmega16d4", "avrxmega2", "avrxmega2"},
+    {"atxmega32a4", "avrxmega2", "avrxmega2"},
+    {"atxmega32a4u", "avrxmega2", "avrxmega2"},
+    {"atxmega32c4", "avrxmega2", "avrxmega2"},
+    {"atxmega32d4", "avrxmega2", "avrxmega2"},
+    {"atxmega32e5", "avrxmega2", "avrxmega2"},
+    {"atxmega16e5", "avrxmega2", "avrxmega2"},
+    {"atxmega8e5", "avrxmega2", "avrxmega2"},
+    {"atxmega64a3u", "avrxmega4", "avrxmega4"},
+    {"atxmega64a4u", "avrxmega4", "avrxmega4"},
+    {"atxmega64b1", "avrxmega4", "avrxmega4"},
+    {"atxmega64b3", "avrxmega4", "avrxmega4"},
+    {"atxmega64c3", "avrxmega4", "avrxmega4"},
+    {"atxmega64d3", "avrxmega4", "avrxmega4"},
+    {"atxmega64d4", "avrxmega4", "avrxmega4"},
+    {"atxmega64a1", "avrxmega5", "avrxmega5"},
+    {"atxmega64a1u", "avrxmega5", "avrxmega5"},
+    {"atxmega128a3", "avrxmega6", "avrxmega6"},
+    {"atxmega128a3u", "avrxmega6", "avrxmega6"},
+    {"atxmega128b1", "avrxmega6", "avrxmega6"},
+    {"atxmega128b3", "avrxmega6", "avrxmega6"},
+    {"atxmega128c3", "avrxmega6", "avrxmega6"},
+    {"atxmega128d3", "avrxmega6", "avrxmega6"},
+    {"atxmega128d4", "avrxmega6", "avrxmega6"},
+    {"atxmega192a3", "avrxmega6", "avrxmega6"},
+    {"atxmega192a3u", "avrxmega6", "avrxmega6"},
+    {"atxmega192c3", "avrxmega6", "avrxmega6"},
+    {"atxmega192d3", "avrxmega6", "avrxmega6"},
+    {"atxmega256a3", "avrxmega6", "avrxmega6"},
+    {"atxmega256a3u", "avrxmega6", "avrxmega6"},
+    {"atxmega256a3b", "avrxmega6", "avrxmega6"},
+    {"atxmega256a3bu", "avrxmega6", "avrxmega6"},
+    {"atxmega256c3", "avrxmega6", "avrxmega6"},
+    {"atxmega256d3", "avrxmega6", "avrxmega6"},
+    {"atxmega384c3", "avrxmega6", "avrxmega6"},
+    {"atxmega384d3", "avrxmega6", "avrxmega6"},
+    {"atxmega128a1", "avrxmega7", "avrxmega7"},
+    {"atxmega128a1u", "avrxmega7", "avrxmega7"},
+    {"atxmega128a4u", "avrxmega7", "avrxmega7"},
 };
 
 std::string GetMCUSubPath(StringRef MCUName) {
@@ -290,11 +289,11 @@ llvm::Optional<StringRef> GetMCUFamilyName(StringRef MCUName) {
   return Optional<StringRef>();
 }
 
-llvm::Optional<unsigned> GetMCUSectionAddressData(StringRef MCUName) {
-  for (const auto &MCU : MCUInfo)
-    if (MCU.Name == MCUName && MCU.DataAddr > 0)
-      return Optional<unsigned>(MCU.DataAddr);
-  return Optional<unsigned>();
+llvm::Optional<unsigned> GetMCUSectionAddressData(StringRef MCU) {
+  return llvm::StringSwitch<llvm::Optional<unsigned>>(MCU)
+      .Case("atmega328", Optional<unsigned>(0x800100))
+      .Case("atmega328p", Optional<unsigned>(0x800100))
+      .Default(Optional<unsigned>());
 }
 
 const StringRef PossibleAVRLibcLocations[] = {
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 2a303dd22062..b6022c60973c 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -258,7 +258,11 @@ static void getExtensionFeatures(const Driver &D,
         << MArch << Error << Ext;
       return;
     }
-    if (isExperimentalExtension(Ext))
+    if (Ext == "zvamo" || Ext == "zvlsseg") {
+      Features.push_back("+experimental-v");
+      Features.push_back("+experimental-zvamo");
+      Features.push_back("+experimental-zvlsseg");
+    } else if (isExperimentalExtension(Ext))
       Features.push_back(Args.MakeArgString("+experimental-" + Ext));
     else
       Features.push_back(Args.MakeArgString("+" + Ext));
@@ -412,9 +416,21 @@ static bool getArchFeatures(const Driver &D, StringRef MArch,
       break;
     case 'b':
       Features.push_back("+experimental-b");
+      Features.push_back("+experimental-zba");
+      Features.push_back("+experimental-zbb");
+      Features.push_back("+experimental-zbc");
+      Features.push_back("+experimental-zbe");
+      Features.push_back("+experimental-zbf");
+      Features.push_back("+experimental-zbm");
+      Features.push_back("+experimental-zbp");
+      Features.push_back("+experimental-zbr");
+      Features.push_back("+experimental-zbs");
+      Features.push_back("+experimental-zbt");
       break;
     case 'v':
       Features.push_back("+experimental-v");
+      Features.push_back("+experimental-zvamo");
+      Features.push_back("+experimental-zvlsseg");
       break;
     }
 
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 110e1a726f55..cd1c6e4f6023 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -128,6 +128,21 @@ template <> struct ScalarEnumerationTraits<FormatStyle::ShortFunctionStyle> {
   }
 };
 
+template <> struct ScalarEnumerationTraits<FormatStyle::AlignConsecutiveStyle> {
+  static void enumeration(IO &IO, FormatStyle::AlignConsecutiveStyle &Value) {
+    IO.enumCase(Value, "None", FormatStyle::ACS_None);
+    IO.enumCase(Value, "Consecutive", FormatStyle::ACS_Consecutive);
+    IO.enumCase(Value, "AcrossEmptyLines", FormatStyle::ACS_AcrossEmptyLines);
+    IO.enumCase(Value, "AcrossComments", FormatStyle::ACS_AcrossComments);
+    IO.enumCase(Value, "AcrossEmptyLinesAndComments",
+                FormatStyle::ACS_AcrossEmptyLinesAndComments);
+
+    // For backward compability.
+    IO.enumCase(Value, "true", FormatStyle::ACS_Consecutive);
+    IO.enumCase(Value, "false", FormatStyle::ACS_None);
+  }
+};
+
 template <> struct ScalarEnumerationTraits<FormatStyle::ShortIfStyle> {
   static void enumeration(IO &IO, FormatStyle::ShortIfStyle &Value) {
     IO.enumCase(Value, "Never", FormatStyle::SIS_Never);
@@ -857,10 +872,10 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.AlignAfterOpenBracket = FormatStyle::BAS_Align;
   LLVMStyle.AlignOperands = FormatStyle::OAS_Align;
   LLVMStyle.AlignTrailingComments = true;
-  LLVMStyle.AlignConsecutiveAssignments = false;
-  LLVMStyle.AlignConsecutiveBitFields = false;
-  LLVMStyle.AlignConsecutiveDeclarations = false;
-  LLVMStyle.AlignConsecutiveMacros = false;
+  LLVMStyle.AlignConsecutiveAssignments = FormatStyle::ACS_None;
+  LLVMStyle.AlignConsecutiveBitFields = FormatStyle::ACS_None;
+  LLVMStyle.AlignConsecutiveDeclarations = FormatStyle::ACS_None;
+  LLVMStyle.AlignConsecutiveMacros = FormatStyle::ACS_None;
   LLVMStyle.AllowAllArgumentsOnNextLine = true;
   LLVMStyle.AllowAllConstructorInitializersOnNextLine = true;
   LLVMStyle.AllowAllParametersOfDeclarationOnNextLine = true;
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index 8cacc5ad4cf4..7d6964b7c72f 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -361,9 +361,10 @@ AlignTokenSequence(unsigned Start, unsigned End, unsigned Column, F &&Matches,
 // that are split across multiple lines. See the test case in FormatTest.cpp
 // that mentions "split function parameter alignment" for an example of this.
 template <typename F>
-static unsigned AlignTokens(const FormatStyle &Style, F &&Matches,
-                            SmallVector<WhitespaceManager::Change, 16> &Changes,
-                            unsigned StartAt) {
+static unsigned AlignTokens(
+    const FormatStyle &Style, F &&Matches,
+    SmallVector<WhitespaceManager::Change, 16> &Changes, unsigned StartAt,
+    const FormatStyle::AlignConsecutiveStyle &ACS = FormatStyle::ACS_None) {
   unsigned MinColumn = 0;
   unsigned MaxColumn = UINT_MAX;
 
@@ -386,6 +387,9 @@ static unsigned AlignTokens(const FormatStyle &Style, F &&Matches,
   // Whether a matching token has been found on the current line.
   bool FoundMatchOnLine = false;
 
+  // Whether the current line consists purely of comments.
+  bool LineIsComment = true;
+
   // Aligns a sequence of matching tokens, on the MinColumn column.
   //
   // Sequences start from the first matching token to align, and end at the
@@ -411,19 +415,38 @@ static unsigned AlignTokens(const FormatStyle &Style, F &&Matches,
     if (Changes[i].NewlinesBefore != 0) {
       CommasBeforeMatch = 0;
       EndOfSequence = i;
-      // If there is a blank line, or if the last line didn't contain any
-      // matching token, the sequence ends here.
-      if (Changes[i].NewlinesBefore > 1 || !FoundMatchOnLine)
+
+      // Whether to break the alignment sequence because of an empty line.
+      bool EmptyLineBreak =
+          (Changes[i].NewlinesBefore > 1) &&
+          (ACS != FormatStyle::ACS_AcrossEmptyLines) &&
+          (ACS != FormatStyle::ACS_AcrossEmptyLinesAndComments);
+
+      // Whether to break the alignment sequence because of a line without a
+      // match.
+      bool NoMatchBreak =
+          !FoundMatchOnLine &&
+          !(LineIsComment &&
+            ((ACS == FormatStyle::ACS_AcrossComments) ||
+             (ACS == FormatStyle::ACS_AcrossEmptyLinesAndComments)));
+
+      if (EmptyLineBreak || NoMatchBreak)
         AlignCurrentSequence();
 
+      // A new line starts, re-initialize line status tracking bools.
       FoundMatchOnLine = false;
+      LineIsComment = true;
+    }
+
+    if (!Changes[i].Tok->is(tok::comment)) {
+      LineIsComment = false;
     }
 
     if (Changes[i].Tok->is(tok::comma)) {
       ++CommasBeforeMatch;
     } else if (Changes[i].indentAndNestingLevel() > IndentAndNestingLevel) {
       // Call AlignTokens recursively, skipping over this scope block.
-      unsigned StoppedAt = AlignTokens(Style, Matches, Changes, i);
+      unsigned StoppedAt = AlignTokens(Style, Matches, Changes, i, ACS);
       i = StoppedAt - 1;
       continue;
     }
@@ -518,7 +541,7 @@ static void AlignMacroSequence(
 }
 
 void WhitespaceManager::alignConsecutiveMacros() {
-  if (!Style.AlignConsecutiveMacros)
+  if (Style.AlignConsecutiveMacros == FormatStyle::ACS_None)
     return;
 
   auto AlignMacrosMatches = [](const Change &C) {
@@ -560,17 +583,41 @@ void WhitespaceManager::alignConsecutiveMacros() {
   // Whether a matching token has been found on the current line.
   bool FoundMatchOnLine = false;
 
+  // Whether the current line consists only of comments
+  bool LineIsComment = true;
+
   unsigned I = 0;
   for (unsigned E = Changes.size(); I != E; ++I) {
     if (Changes[I].NewlinesBefore != 0) {
       EndOfSequence = I;
-      // If there is a blank line, or if the last line didn't contain any
-      // matching token, the sequence ends here.
-      if (Changes[I].NewlinesBefore > 1 || !FoundMatchOnLine)
+
+      // Whether to break the alignment sequence because of an empty line.
+      bool EmptyLineBreak =
+          (Changes[I].NewlinesBefore > 1) &&
+          (Style.AlignConsecutiveMacros != FormatStyle::ACS_AcrossEmptyLines) &&
+          (Style.AlignConsecutiveMacros !=
+           FormatStyle::ACS_AcrossEmptyLinesAndComments);
+
+      // Whether to break the alignment sequence because of a line without a
+      // match.
+      bool NoMatchBreak =
+          !FoundMatchOnLine &&
+          !(LineIsComment && ((Style.AlignConsecutiveMacros ==
+                               FormatStyle::ACS_AcrossComments) ||
+                              (Style.AlignConsecutiveMacros ==
+                               FormatStyle::ACS_AcrossEmptyLinesAndComments)));
+
+      if (EmptyLineBreak || NoMatchBreak)
         AlignMacroSequence(StartOfSequence, EndOfSequence, MinColumn, MaxColumn,
                            FoundMatchOnLine, AlignMacrosMatches, Changes);
 
+      // A new line starts, re-initialize line status tracking bools.
       FoundMatchOnLine = false;
+      LineIsComment = true;
+    }
+
+    if (!Changes[I].Tok->is(tok::comment)) {
+      LineIsComment = false;
     }
 
     if (!AlignMacrosMatches(Changes[I]))
@@ -597,7 +644,7 @@ void WhitespaceManager::alignConsecutiveMacros() {
 }
 
 void WhitespaceManager::alignConsecutiveAssignments() {
-  if (!Style.AlignConsecutiveAssignments)
+  if (Style.AlignConsecutiveAssignments == FormatStyle::ACS_None)
     return;
 
   AlignTokens(
@@ -613,11 +660,11 @@ void WhitespaceManager::alignConsecutiveAssignments() {
 
         return C.Tok->is(tok::equal);
       },
-      Changes, /*StartAt=*/0);
+      Changes, /*StartAt=*/0, Style.AlignConsecutiveAssignments);
 }
 
 void WhitespaceManager::alignConsecutiveBitFields() {
-  if (!Style.AlignConsecutiveBitFields)
+  if (Style.AlignConsecutiveBitFields == FormatStyle::ACS_None)
     return;
 
   AlignTokens(
@@ -633,11 +680,11 @@ void WhitespaceManager::alignConsecutiveBitFields() {
 
         return C.Tok->is(TT_BitFieldColon);
       },
-      Changes, /*StartAt=*/0);
+      Changes, /*StartAt=*/0, Style.AlignConsecutiveBitFields);
 }
 
 void WhitespaceManager::alignConsecutiveDeclarations() {
-  if (!Style.AlignConsecutiveDeclarations)
+  if (Style.AlignConsecutiveDeclarations == FormatStyle::ACS_None)
     return;
 
   // FIXME: Currently we don't handle properly the PointerAlignment: Right
@@ -670,7 +717,7 @@ void WhitespaceManager::alignConsecutiveDeclarations() {
         }
         return true;
       },
-      Changes, /*StartAt=*/0);
+      Changes, /*StartAt=*/0, Style.AlignConsecutiveDeclarations);
 }
 
 void WhitespaceManager::alignChainedConditionals() {
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 933a91877c9b..20edcd624a18 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -1140,10 +1140,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
 
   // OpenCL definitions.
   if (LangOpts.OpenCL) {
-#define OPENCLEXT(Ext)                                                         \
-  if (TI.getSupportedOpenCLOpts().isSupported(#Ext, LangOpts))                 \
-    Builder.defineMacro(#Ext);
-#include "clang/Basic/OpenCLExtensions.def"
+    TI.getOpenCLFeatureDefines(LangOpts, Builder);
 
     if (TI.getTriple().isSPIR())
       Builder.defineMacro("__IMAGE_SUPPORT__");
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 851fde998942..f9b852826775 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -790,7 +790,7 @@ void Parser::HandlePragmaOpenCLExtension() {
     PP.Diag(NameLoc, diag::warn_pragma_unknown_extension) << Ident;
   else if (Opt.isSupportedExtension(Name, getLangOpts()))
     Opt.enable(Name, State == Enable);
-  else if (Opt.isSupportedCore(Name, getLangOpts()))
+  else if (Opt.isSupportedCoreOrOptionalCore(Name, getLangOpts()))
     PP.Diag(NameLoc, diag::warn_pragma_extension_is_core) << Ident;
   else
     PP.Diag(NameLoc, diag::warn_pragma_unsupported_extension) << Ident;
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index b62e1e10d469..10d45930eec5 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -325,7 +325,7 @@ void Sema::Initialize() {
   // core features.
   if (getLangOpts().OpenCL) {
     getOpenCLOptions().addSupport(
-        Context.getTargetInfo().getSupportedOpenCLOpts());
+        Context.getTargetInfo().getSupportedOpenCLOpts(), getLangOpts());
     getOpenCLOptions().enableSupportedCore(getLangOpts());
     addImplicitTypedef("sampler_t", Context.OCLSamplerTy);
     addImplicitTypedef("event_t", Context.OCLEventTy);
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index d77c9e43a9bd..c2785fd60fc2 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -3529,9 +3529,11 @@ CodeCompletionString *CodeCompletionResult::createCodeCompletionStringForDecl(
         Result.AddTypedTextChunk("");
     }
     unsigned Idx = 0;
+    // The extra Idx < Sel.getNumArgs() check is needed due to legacy C-style
+    // method parameters.
     for (ObjCMethodDecl::param_const_iterator P = Method->param_begin(),
                                               PEnd = Method->param_end();
-         P != PEnd; (void)++P, ++Idx) {
+         P != PEnd && Idx < Sel.getNumArgs(); (void)++P, ++Idx) {
       if (Idx > 0) {
         std::string Keyword;
         if (Idx > StartParameter)
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index efb3e77e2f50..f83bda22e171 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4364,6 +4364,9 @@ TemplateDeclInstantiator::SubstFunctionType(FunctionDecl *D,
       for (unsigned OldIdx = 0, NumOldParams = OldProtoLoc.getNumParams();
            OldIdx != NumOldParams; ++OldIdx) {
         ParmVarDecl *OldParam = OldProtoLoc.getParam(OldIdx);
+        if (!OldParam)
+          return nullptr;
+
         LocalInstantiationScope *Scope = SemaRef.CurrentInstantiationScope;
 
         Optional<unsigned> NumArgumentsInExpansion;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 4a453bfe3e63..76bd84ceb036 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -3609,11 +3609,12 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
     case OPENCL_EXTENSIONS:
       for (unsigned I = 0, E = Record.size(); I != E; ) {
         auto Name = ReadString(Record, I);
-        auto &Opt = OpenCLExtensions.OptMap[Name];
-        Opt.Supported = Record[I++] != 0;
-        Opt.Enabled = Record[I++] != 0;
-        Opt.Avail = Record[I++];
-        Opt.Core = Record[I++];
+        auto &OptInfo = OpenCLExtensions.OptMap[Name];
+        OptInfo.Supported = Record[I++] != 0;
+        OptInfo.Enabled = Record[I++] != 0;
+        OptInfo.Avail = Record[I++];
+        OptInfo.Core = Record[I++];
+        OptInfo.Opt = Record[I++];
       }
       break;
 
@@ -7875,7 +7876,7 @@ void ASTReader::InitializeSema(Sema &S) {
         NewOverrides.applyOverrides(SemaObj->getLangOpts());
   }
 
-  SemaObj->OpenCLFeatures.copy(OpenCLExtensions);
+  SemaObj->OpenCLFeatures = OpenCLExtensions;
   SemaObj->OpenCLTypeExtMap = OpenCLTypeExtMap;
   SemaObj->OpenCLDeclExtMap = OpenCLDeclExtMap;
 
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 9c16ded13cd9..6bfa7b0e7d6d 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3978,6 +3978,7 @@ void ASTWriter::WriteOpenCLExtensions(Sema &SemaRef) {
     Record.push_back(V.Enabled ? 1 : 0);
     Record.push_back(V.Avail);
     Record.push_back(V.Core);
+    Record.push_back(V.Opt);
   }
   Stream.EmitRecord(OPENCL_EXTENSIONS, Record);
 }
diff --git a/clang/test/CodeGen/basic-block-sections.c b/clang/test/CodeGen/basic-block-sections.c
index 91bcb6222848..70cdeeebb0d3 100644
--- a/clang/test/CodeGen/basic-block-sections.c
+++ b/clang/test/CodeGen/basic-block-sections.c
@@ -41,4 +41,4 @@ int another(int a) {
 //
 // UNIQUE: .section .text.world.world.__part.1,
 // UNIQUE: .section .text.another.another.__part.1,
-// ERROR: error:  unable to load basic block sections function list: '{{.*}}{{[Nn]}}o such file or directory'
+// ERROR: error:  unable to load basic block sections function list: '{{[Nn]}}o such file or directory'
diff --git a/clang/test/CodeGen/ubsan-blacklist-vfs.c b/clang/test/CodeGen/ubsan-blacklist-vfs.c
index a8509eb2f6f9..0647cbf7a57b 100644
--- a/clang/test/CodeGen/ubsan-blacklist-vfs.c
+++ b/clang/test/CodeGen/ubsan-blacklist-vfs.c
@@ -8,10 +8,10 @@
 // RUN: %clang_cc1 -fsanitize=unsigned-integer-overflow -ivfsoverlay %t-vfsoverlay.yaml -fsanitize-blacklist=%/T/only-virtual-file.blacklist -emit-llvm %s -o - | FileCheck %s --check-prefix=FUNC
 
 // RUN: not %clang_cc1 -fsanitize=unsigned-integer-overflow -ivfsoverlay %t-vfsoverlay.yaml -fsanitize-blacklist=%/T/invalid-virtual-file.blacklist -emit-llvm %s -o - 2>&1 | FileCheck %s --check-prefix=INVALID-MAPPED-FILE
-// INVALID-MAPPED-FILE: invalid-virtual-file.blacklist': {{.*}}{{[Nn]}}o such file or directory
+// INVALID-MAPPED-FILE: invalid-virtual-file.blacklist': {{[Nn]}}o such file or directory
 
 // RUN: not %clang_cc1 -fsanitize=unsigned-integer-overflow -ivfsoverlay %t-vfsoverlay.yaml -fsanitize-blacklist=%t-nonexistent.blacklist -emit-llvm %s -o - 2>&1 | FileCheck %s --check-prefix=INVALID
-// INVALID: nonexistent.blacklist': {{.*}}{{[Nn]}}o such file or directory
+// INVALID: nonexistent.blacklist': {{[Nn]}}o such file or directory
 
 unsigned i;
 
diff --git a/clang/test/CodeGenCXX/ms-lookup-template-base-classes.cpp b/clang/test/CodeGenCXX/ms-lookup-template-base-classes.cpp
index fbc211aca17a..41a4ee8b5bb8 100644
--- a/clang/test/CodeGenCXX/ms-lookup-template-base-classes.cpp
+++ b/clang/test/CodeGenCXX/ms-lookup-template-base-classes.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple x86_64-windows-msvc -emit-obj -fms-compatibility %s -o -
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -emit-llvm -fms-compatibility %s -o -
 // CHECK that we don't crash.
 
 struct Base {
diff --git a/clang/test/Driver/Inputs/basic_avr_tree/bin/avr-ld b/clang/test/Driver/Inputs/basic_avr_tree/bin/avr-ld
deleted file mode 100755
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_avr_tree/lib/avr/lib/libavr.a b/clang/test/Driver/Inputs/basic_avr_tree/lib/avr/lib/libavr.a
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_avr_tree/lib/gcc/avr/5.4.0/libgcc.a b/clang/test/Driver/Inputs/basic_avr_tree/lib/gcc/avr/5.4.0/libgcc.a
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/avr-ld.c b/clang/test/Driver/avr-ld.c
deleted file mode 100644
index 535990bf9570..000000000000
--- a/clang/test/Driver/avr-ld.c
+++ /dev/null
@@ -1,44 +0,0 @@
-// RUN: %clang -### --target=avr -mmcu=at90s2313 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKA %s
-// LINKA: {{".*ld.*"}} {{.*}} {{"-L.*tiny-stack"}} {{.*}} "-Tdata=0x800060" {{.*}} "-lat90s2313" "-mavr2"
-
-// RUN: %clang -### --target=avr -mmcu=at90s8515 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKB %s
-// LINKB: {{".*ld.*"}} {{.*}} "-Tdata=0x800060" {{.*}} "-lat90s8515" "-mavr2"
-
-// RUN: %clang -### --target=avr -mmcu=attiny13 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKC %s
-// LINKC: {{".*ld.*"}} {{.*}} {{"-L.*avr25/tiny-stack"}} {{.*}} "-Tdata=0x800060" {{.*}} "-lattiny13" "-mavr25"
-
-// RUN: %clang -### --target=avr -mmcu=attiny44 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKD %s
-// LINKD: {{".*ld.*"}} {{.*}} {{"-L.*avr25"}} {{.*}} "-Tdata=0x800060" {{.*}} "-lattiny44" "-mavr25"
-
-// RUN: %clang -### --target=avr -mmcu=atmega103 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKE %s
-// LINKE: {{".*ld.*"}} {{.*}} {{"-L.*avr31"}} {{.*}} "-Tdata=0x800060" {{.*}} "-latmega103" "-mavr31"
-
-// RUN: %clang -### --target=avr -mmcu=atmega8u2 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKF %s
-// LINKF: {{".*ld.*"}} {{.*}} {{"-L.*avr35"}} {{.*}} "-Tdata=0x800100" {{.*}} "-latmega8u2" "-mavr35"
-
-// RUN: %clang -### --target=avr -mmcu=atmega48pa --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKG %s
-// LINKG: {{".*ld.*"}} {{.*}} {{"-L.*avr4"}} {{.*}} "-Tdata=0x800100" {{.*}} "-latmega48pa" "-mavr4"
-
-// RUN: %clang -### --target=avr -mmcu=atmega328 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKH %s
-// LINKH: {{".*ld.*"}} {{.*}} {{"-L.*avr5"}} {{.*}} "-Tdata=0x800100" {{.*}} "-latmega328" "-mavr5"
-
-// RUN: %clang -### --target=avr -mmcu=atmega1281 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKI %s
-// LINKI: {{".*ld.*"}} {{.*}} {{"-L.*avr51"}} {{.*}} "-Tdata=0x800200" {{.*}} "-latmega1281" "-mavr51"
-
-// RUN: %clang -### --target=avr -mmcu=atmega2560 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKJ %s
-// LINKJ: {{".*ld.*"}} {{.*}} {{"-L.*avr6"}} {{.*}} "-Tdata=0x800200" {{.*}} "-latmega2560" "-mavr6"
-
-// RUN: %clang -### --target=avr -mmcu=attiny10 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKK %s
-// LINKK: {{".*ld.*"}} {{.*}} {{"-L.*avrtiny"}} {{.*}} "-Tdata=0x800040" {{.*}} "-lattiny10" "-mavrtiny"
-
-// RUN: %clang -### --target=avr -mmcu=atxmega16a4 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKL %s
-// LINKL: {{".*ld.*"}} {{.*}} {{"-L.*avrxmega2"}} {{.*}} "-Tdata=0x802000" {{.*}} "-latxmega16a4" "-mavrxmega2"
-
-// RUN: %clang -### --target=avr -mmcu=atxmega64b3 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKM %s
-// LINKM: {{".*ld.*"}} {{.*}} {{"-L.*avrxmega4"}} {{.*}} "-Tdata=0x802000" {{.*}} "-latxmega64b3" "-mavrxmega4"
-
-// RUN: %clang -### --target=avr -mmcu=atxmega128a3u --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKN %s
-// LINKN: {{".*ld.*"}} {{.*}} {{"-L.*avrxmega6"}} {{.*}} "-Tdata=0x802000" {{.*}} "-latxmega128a3u" "-mavrxmega6"
-
-// RUN: %clang -### --target=avr -mmcu=atxmega128a1 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKO %s
-// LINKO: {{".*ld.*"}} {{.*}} {{"-L.*avrxmega7"}} {{.*}} "-Tdata=0x802000" {{.*}} "-latxmega128a1" "-mavrxmega7"
diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c
index 84b48a2af059..53585be7477c 100644
--- a/clang/test/Driver/clang-offload-bundler.c
+++ b/clang/test/Driver/clang-offload-bundler.c
@@ -75,7 +75,7 @@
 
 // RUN: not clang-offload-bundler -type=i -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2.notexist -outputs=%t.bundle.i 2>&1 | FileCheck %s -DFILE=%t.tgt2.notexist --check-prefix CK-ERR5
 // RUN: not clang-offload-bundler -type=i -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.i,%t.tgt1,%t.tgt2 -inputs=%t.bundle.i.notexist -unbundle 2>&1 | FileCheck %s -DFILE=%t.bundle.i.notexist --check-prefix CK-ERR5
-// CK-ERR5: error: '[[FILE]]': {{.*}}{{N|n}}o such file or directory
+// CK-ERR5: error: '[[FILE]]': {{N|n}}o such file or directory
 
 // RUN: not clang-offload-bundler -type=invalid -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s -DTYPE=invalid --check-prefix CK-ERR6
 // CK-ERR6: error: '[[TYPE]]': invalid file type specified
diff --git a/clang/test/Frontend/output-failures.c b/clang/test/Frontend/output-failures.c
index 26d25fdbef36..0e9ea63f8067 100644
--- a/clang/test/Frontend/output-failures.c
+++ b/clang/test/Frontend/output-failures.c
@@ -1,4 +1,4 @@
 // RUN: not %clang_cc1 -emit-llvm -o %t.doesnotexist/somename %s 2> %t
 // RUN: FileCheck -check-prefix=OUTPUTFAIL -input-file=%t %s
 
-// OUTPUTFAIL: error: unable to open output file '{{.*}}doesnotexist{{.}}somename': '{{.*}}{{[nN]}}o such file or directory{{.*}}'
+// OUTPUTFAIL: error: unable to open output file '{{.*}}doesnotexist{{.}}somename': '{{[nN]}}o such file or directory'
diff --git a/clang/test/Frontend/stats-file.c b/clang/test/Frontend/stats-file.c
index 0d0a97b0d1d6..53b264534c57 100644
--- a/clang/test/Frontend/stats-file.c
+++ b/clang/test/Frontend/stats-file.c
@@ -5,4 +5,4 @@
 // CHECK: }
 
 // RUN: %clang_cc1 -emit-llvm -o %t -stats-file=%t.doesnotexist/bla %s 2>&1 | FileCheck -check-prefix=OUTPUTFAIL %s
-// OUTPUTFAIL: warning: unable to open statistics output file '{{.*}}doesnotexist{{.}}bla': '{{.*}}{{[Nn]}}o such file or directory{{.*}}'
+// OUTPUTFAIL: warning: unable to open statistics output file '{{.*}}doesnotexist{{.}}bla': '{{[Nn]}}o such file or directory'
diff --git a/clang/test/Misc/nvptx.languageOptsOpenCL.cl b/clang/test/Misc/nvptx.languageOptsOpenCL.cl
index 36cbfeac3754..fd75981b219a 100644
--- a/clang/test/Misc/nvptx.languageOptsOpenCL.cl
+++ b/clang/test/Misc/nvptx.languageOptsOpenCL.cl
@@ -2,19 +2,15 @@
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple nvptx-unknown-unknown
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple nvptx-unknown-unknown
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple nvptx-unknown-unknown
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple nvptx-unknown-unknown
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple nvptx-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple nvptx-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple nvptx-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple nvptx-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple nvptx64-unknown-unknown
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple nvptx64-unknown-unknown
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple nvptx64-unknown-unknown
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple nvptx64-unknown-unknown
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple nvptx64-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple nvptx64-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple nvptx64-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple nvptx64-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 
 // Extensions in all versions
 #ifndef cl_clang_storage_class_specifiers
diff --git a/clang/test/Misc/r600.languageOptsOpenCL.cl b/clang/test/Misc/r600.languageOptsOpenCL.cl
index 253e4fdcbc59..2c87370f9d5f 100644
--- a/clang/test/Misc/r600.languageOptsOpenCL.cl
+++ b/clang/test/Misc/r600.languageOptsOpenCL.cl
@@ -2,27 +2,21 @@
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple r600-unknown-unknown -target-cpu cayman
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple r600-unknown-unknown -target-cpu cayman
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple r600-unknown-unknown -target-cpu cayman
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple r600-unknown-unknown -target-cpu cayman
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cayman
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cayman
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cayman
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cayman
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple r600-unknown-unknown -target-cpu cypress
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple r600-unknown-unknown -target-cpu cypress
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple r600-unknown-unknown -target-cpu cypress
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple r600-unknown-unknown -target-cpu cypress
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cypress
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cypress
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cypress
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu cypress
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple r600-unknown-unknown -target-cpu turks
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple r600-unknown-unknown -target-cpu turks
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple r600-unknown-unknown -target-cpu turks
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple r600-unknown-unknown -target-cpu turks
 // RUN: %clang_cc1 -x cl -cl-std=CL %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu turks
 // RUN: %clang_cc1 -x cl -cl-std=CL1.1 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu turks
 // RUN: %clang_cc1 -x cl -cl-std=CL1.2 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu turks
-// RUN: %clang_cc1 -x cl -cl-std=CL2.0 %s -verify -triple r600-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES -target-cpu turks
 
 // Extensions in all versions
 #ifndef cl_clang_storage_class_specifiers
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 12a33e0aa18a..93f13895ec13 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -12,12 +12,27 @@
 // CHECK-NOT: __riscv_fdiv
 // CHECK-NOT: __riscv_fsqrt
 // CHECK-NOT: __riscv_atomic
+// CHECK-NOT: __riscv_zba
+// CHECK-NOT: __riscv_zbb
+// CHECK-NOT: __riscv_zbc
+// CHECK-NOT: __riscv_zbe
+// CHECK-NOT: __riscv_zbf
+// CHECK-NOT: __riscv_zbm
+// CHECK-NOT: __riscv_zbp
+// CHECK-NOT: __riscv_zbproposedc
+// CHECK-NOT: __riscv_zbr
+// CHECK-NOT: __riscv_zbs
+// CHECK-NOT: __riscv_zbt
+// CHECK-NOT: __riscv_zfh
+// CHECK-NOT: __riscv_zvamo
+// CHECK-NOT: __riscv_zvlsseg
 
 // RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32im -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-M-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64im -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-M-EXT %s
 // CHECK-M-EXT: __riscv_div 1
+// CHECK-M-EXT: __riscv_m 2000000
 // CHECK-M-EXT: __riscv_mul 1
 // CHECK-M-EXT: __riscv_muldiv 1
 
@@ -25,12 +40,14 @@
 // RUN: -o - | FileCheck --check-prefix=CHECK-A-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64ia -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-A-EXT %s
+// CHECK-A-EXT: __riscv_a 2000000
 // CHECK-A-EXT: __riscv_atomic 1
 
 // RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32if -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-F-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64if -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-F-EXT %s
+// CHECK-F-EXT: __riscv_f 2000000
 // CHECK-F-EXT: __riscv_fdiv 1
 // CHECK-F-EXT: __riscv_flen 32
 // CHECK-F-EXT: __riscv_fsqrt 1
@@ -39,6 +56,7 @@
 // RUN: -o - | FileCheck --check-prefix=CHECK-D-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64ifd -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-D-EXT %s
+// CHECK-D-EXT: __riscv_d 2000000
 // CHECK-D-EXT: __riscv_fdiv 1
 // CHECK-D-EXT: __riscv_flen 64
 // CHECK-D-EXT: __riscv_fsqrt 1
@@ -47,13 +65,25 @@
 // RUN: -o - | FileCheck --check-prefix=CHECK-C-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64ic -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-C-EXT %s
+// CHECK-C-EXT: __riscv_c 2000000
 // CHECK-C-EXT: __riscv_compressed 1
 
 // RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32ib0p93 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-B-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64ib0p93 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-B-EXT %s
+// CHECK-B-EXT: __riscv_b 93000
 // CHECK-B-EXT: __riscv_bitmanip 1
+// CHECK-B-EXT: __riscv_zba 93000
+// CHECK-B-EXT: __riscv_zbb 93000
+// CHECK-B-EXT: __riscv_zbc 93000
+// CHECK-B-EXT: __riscv_zbe 93000
+// CHECK-B-EXT: __riscv_zbf 93000
+// CHECK-B-EXT: __riscv_zbm 93000
+// CHECK-B-EXT: __riscv_zbp 93000
+// CHECK-B-EXT: __riscv_zbr 93000
+// CHECK-B-EXT: __riscv_zbs 93000
+// CHECK-B-EXT: __riscv_zbt 93000
 
 // RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32ifd -mabi=ilp32 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-SOFT %s
@@ -85,10 +115,98 @@
 // RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions \
 // RUN:   -march=rv64iv0p9 -x c -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izvamo0p9 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv32izvamo0p9 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izvlsseg0p9 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv32izvlsseg0p9 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// CHECK-V-EXT: __riscv_v 9000
 // CHECK-V-EXT: __riscv_vector 1
-//
+// CHECK-V-EXT: __riscv_zvamo 9000
+// CHECK-V-EXT: __riscv_zvlsseg 9000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izba0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBA-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izba0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBA-EXT %s
+// CHECK-ZBA-NOT: __riscv_b
+// CHECK-ZBA-EXT: __riscv_zba 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbb0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBB-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbb0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBB-EXT %s
+// CHECK-ZBB-NOT: __riscv_b
+// CHECK-ZBB-EXT: __riscv_zbb 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbc0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBC-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbc0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBC-EXT %s
+// CHECK-ZBC-NOT: __riscv_b
+// CHECK-ZBC-EXT: __riscv_zbc 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbe0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBE-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbe0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBE-EXT %s
+// CHECK-ZBE-NOT: __riscv_b
+// CHECK-ZBE-EXT: __riscv_zbe 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbf0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBF-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbf0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBF-EXT %s
+// CHECK-ZBF-NOT: __riscv_b
+// CHECK-ZBF-EXT: __riscv_zbf 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbm0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBM-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbm0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBM-EXT %s
+// CHECK-ZBM-NOT: __riscv_b
+// CHECK-ZBM-EXT: __riscv_zbm 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbp0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBP-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbp0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBP-EXT %s
+// CHECK-ZBP-NOT: __riscv_b
+// CHECK-ZBP-EXT: __riscv_zbp 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbproposedc0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBPROPOSEDC-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbproposedc0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBPROPOSEDC-EXT %s
+// CHECK-ZBPROPOSEDC-NOT: __riscv_b
+// CHECK-ZBPROPOSEDC-EXT: __riscv_zbproposedc 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbr0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBR-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbr0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBR-EXT %s
+// CHECK-ZBR-NOT: __riscv_b
+// CHECK-ZBR-EXT: __riscv_zbr 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbs0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBS-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbs0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBS-EXT %s
+// CHECK-ZBS-NOT: __riscv_b
+// CHECK-ZBS-EXT: __riscv_zbs 93000
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izbt0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBT-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izbt0p93 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZBT-EXT %s
+// CHECK-ZBT-NOT: __riscv_b
+// CHECK-ZBT-EXT: __riscv_zbt 93000
+
 // RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izfh0p1 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-ZFH-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izfh0p1 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-ZFH-EXT %s
-// CHECK-ZFH-EXT: __riscv_zfh 1
+// CHECK-ZFH-EXT: __riscv_zfh 1000
diff --git a/clang/test/SemaCXX/subst-func-type-invalid-ret-type.cpp b/clang/test/SemaCXX/subst-func-type-invalid-ret-type.cpp
new file mode 100644
index 000000000000..c78ffff3ff50
--- /dev/null
+++ b/clang/test/SemaCXX/subst-func-type-invalid-ret-type.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang -fsyntax-only -std=c++17 %s -Xclang -verify
+
+// The important part is that we do not crash.
+
+template<typename T> T declval();
+
+template <typename T>
+auto Call(T x) -> decltype(declval<T>()(0)) {} // expected-note{{candidate template ignored}}
+
+class Status {};
+
+void fun() {
+  // The Status() (instead of Status) here used to cause a crash.
+  Call([](auto x) -> Status() {}); // expected-error{{function cannot return function type 'Status ()}}
+  // expected-error@-1{{no matching function for call to 'Call'}}
+}
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 1565016802f9..0c37d1188f15 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -11395,8 +11395,8 @@ TEST_F(FormatTest, ConfigurableUseOfTab) {
                    "*/\n"
                    "}",
                    Tab));
-  Tab.AlignConsecutiveAssignments = true;
-  Tab.AlignConsecutiveDeclarations = true;
+  Tab.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
+  Tab.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   Tab.TabWidth = 4;
   Tab.IndentWidth = 4;
   verifyFormat("class Assign {\n"
@@ -11634,8 +11634,8 @@ TEST_F(FormatTest, ConfigurableUseOfTab) {
                    "*/\n"
                    "}",
                    Tab));
-  Tab.AlignConsecutiveAssignments = true;
-  Tab.AlignConsecutiveDeclarations = true;
+  Tab.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
+  Tab.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   Tab.TabWidth = 4;
   Tab.IndentWidth = 4;
   verifyFormat("class Assign {\n"
@@ -12470,9 +12470,9 @@ TEST_F(FormatTest, ConfigurableSpaceAroundPointerQualifiers) {
 
 TEST_F(FormatTest, AlignConsecutiveMacros) {
   FormatStyle Style = getLLVMStyle();
-  Style.AlignConsecutiveAssignments = true;
-  Style.AlignConsecutiveDeclarations = true;
-  Style.AlignConsecutiveMacros = false;
+  Style.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
+  Style.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_None;
 
   verifyFormat("#define a 3\n"
                "#define bbbb 4\n"
@@ -12496,7 +12496,7 @@ TEST_F(FormatTest, AlignConsecutiveMacros) {
                "#define ffff(x, y) (x - y)",
                Style);
 
-  Style.AlignConsecutiveMacros = true;
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
   verifyFormat("#define a    3\n"
                "#define bbbb 4\n"
                "#define ccc  (5)",
@@ -12536,7 +12536,7 @@ TEST_F(FormatTest, AlignConsecutiveMacros) {
                "};",
                Style);
 
-  Style.AlignConsecutiveMacros = false;
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_None;
   Style.ColumnLimit = 20;
 
   verifyFormat("#define a          \\\n"
@@ -12550,7 +12550,7 @@ TEST_F(FormatTest, AlignConsecutiveMacros) {
                "  \"LLLLLLLL\"\n",
                Style);
 
-  Style.AlignConsecutiveMacros = true;
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
   verifyFormat("#define a          \\\n"
                "  \"aabbbbbbbbbbbb\"\n"
                "#define D          \\\n"
@@ -12561,12 +12561,766 @@ TEST_F(FormatTest, AlignConsecutiveMacros) {
                "  \"FFFFFFFFFFFFF\"  \\\n"
                "  \"LLLLLLLL\"\n",
                Style);
+
+  // Test across comments
+  Style.MaxEmptyLinesToKeep = 10;
+  Style.ReflowComments = false;
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_AcrossComments;
+  EXPECT_EQ("#define a    3\n"
+            "// line comment\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a 3\n"
+                   "// line comment\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a    3\n"
+            "/* block comment */\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a  3\n"
+                   "/* block comment */\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a    3\n"
+            "/* multi-line *\n"
+            " * block comment */\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a 3\n"
+                   "/* multi-line *\n"
+                   " * block comment */\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a    3\n"
+            "// multi-line line comment\n"
+            "//\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a  3\n"
+                   "// multi-line line comment\n"
+                   "//\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a 3\n"
+            "// empty lines still break.\n"
+            "\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a     3\n"
+                   "// empty lines still break.\n"
+                   "\n"
+                   "#define bbbb     4\n"
+                   "#define ccc  (5)",
+                   Style));
+
+  // Test across empty lines
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_AcrossEmptyLines;
+  EXPECT_EQ("#define a    3\n"
+            "\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a 3\n"
+                   "\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a    3\n"
+            "\n"
+            "\n"
+            "\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a        3\n"
+                   "\n"
+                   "\n"
+                   "\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a 3\n"
+            "// comments should break alignment\n"
+            "//\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a        3\n"
+                   "// comments should break alignment\n"
+                   "//\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  // Test across empty lines and comments
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_AcrossEmptyLinesAndComments;
+  verifyFormat("#define a    3\n"
+               "\n"
+               "// line comment\n"
+               "#define bbbb 4\n"
+               "#define ccc  (5)",
+               Style);
+
+  EXPECT_EQ("#define a    3\n"
+            "\n"
+            "\n"
+            "/* multi-line *\n"
+            " * block comment */\n"
+            "\n"
+            "\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a 3\n"
+                   "\n"
+                   "\n"
+                   "/* multi-line *\n"
+                   " * block comment */\n"
+                   "\n"
+                   "\n"
+                   "#define bbbb 4\n"
+                   "#define ccc (5)",
+                   Style));
+
+  EXPECT_EQ("#define a    3\n"
+            "\n"
+            "\n"
+            "/* multi-line *\n"
+            " * block comment */\n"
+            "\n"
+            "\n"
+            "#define bbbb 4\n"
+            "#define ccc  (5)",
+            format("#define a 3\n"
+                   "\n"
+                   "\n"
+                   "/* multi-line *\n"
+                   " * block comment */\n"
+                   "\n"
+                   "\n"
+                   "#define bbbb 4\n"
+                   "#define ccc       (5)",
+                   Style));
+}
+
+TEST_F(FormatTest, AlignConsecutiveAssignmentsAcrossEmptyLines) {
+  FormatStyle Alignment = getLLVMStyle();
+  Alignment.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_AcrossEmptyLines;
+
+  Alignment.MaxEmptyLinesToKeep = 10;
+  /* Test alignment across empty lines */
+  EXPECT_EQ("int a           = 5;\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a       = 5;\n"
+                   "\n"
+                   "int oneTwoThree= 123;",
+                   Alignment));
+  EXPECT_EQ("int a           = 5;\n"
+            "int one         = 1;\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "int one = 1;\n"
+                   "\n"
+                   "int oneTwoThree = 123;",
+                   Alignment));
+  EXPECT_EQ("int a           = 5;\n"
+            "int one         = 1;\n"
+            "\n"
+            "int oneTwoThree = 123;\n"
+            "int oneTwo      = 12;",
+            format("int a = 5;\n"
+                   "int one = 1;\n"
+                   "\n"
+                   "int oneTwoThree = 123;\n"
+                   "int oneTwo = 12;",
+                   Alignment));
+
+  /* Test across comments */
+  EXPECT_EQ("int a = 5;\n"
+            "/* block comment */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a = 5;\n"
+            "// line comment\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  /* Test across comments and newlines */
+  EXPECT_EQ("int a = 5;\n"
+            "\n"
+            "/* block comment */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a = 5;\n"
+            "\n"
+            "// line comment\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+}
+
+TEST_F(FormatTest, AlignConsecutiveDeclarationsAcrossEmptyLinesAndComments) {
+  FormatStyle Alignment = getLLVMStyle();
+  Alignment.AlignConsecutiveDeclarations =
+      FormatStyle::ACS_AcrossEmptyLinesAndComments;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_None;
+
+  Alignment.MaxEmptyLinesToKeep = 10;
+  /* Test alignment across empty lines */
+  EXPECT_EQ("int         a = 5;\n"
+            "\n"
+            "float const oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "float const oneTwoThree = 123;",
+                   Alignment));
+  EXPECT_EQ("int         a = 5;\n"
+            "float const one = 1;\n"
+            "\n"
+            "int         oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "float const one = 1;\n"
+                   "\n"
+                   "int oneTwoThree = 123;",
+                   Alignment));
+
+  /* Test across comments */
+  EXPECT_EQ("float const a = 5;\n"
+            "/* block comment */\n"
+            "int         oneTwoThree = 123;",
+            format("float const a = 5;\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("float const a = 5;\n"
+            "// line comment\n"
+            "int         oneTwoThree = 123;",
+            format("float const a = 5;\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  /* Test across comments and newlines */
+  EXPECT_EQ("float const a = 5;\n"
+            "\n"
+            "/* block comment */\n"
+            "int         oneTwoThree = 123;",
+            format("float const a = 5;\n"
+                   "\n"
+                   "/* block comment */\n"
+                   "int         oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("float const a = 5;\n"
+            "\n"
+            "// line comment\n"
+            "int         oneTwoThree = 123;",
+            format("float const a = 5;\n"
+                   "\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+}
+
+TEST_F(FormatTest, AlignConsecutiveBitFieldsAcrossEmptyLinesAndComments) {
+  FormatStyle Alignment = getLLVMStyle();
+  Alignment.AlignConsecutiveBitFields =
+      FormatStyle::ACS_AcrossEmptyLinesAndComments;
+
+  Alignment.MaxEmptyLinesToKeep = 10;
+  /* Test alignment across empty lines */
+  EXPECT_EQ("int a            : 5;\n"
+            "\n"
+            "int longbitfield : 6;",
+            format("int a : 5;\n"
+                   "\n"
+                   "int longbitfield : 6;",
+                   Alignment));
+  EXPECT_EQ("int a            : 5;\n"
+            "int one          : 1;\n"
+            "\n"
+            "int longbitfield : 6;",
+            format("int a : 5;\n"
+                   "int one : 1;\n"
+                   "\n"
+                   "int longbitfield : 6;",
+                   Alignment));
+
+  /* Test across comments */
+  EXPECT_EQ("int a            : 5;\n"
+            "/* block comment */\n"
+            "int longbitfield : 6;",
+            format("int a : 5;\n"
+                   "/* block comment */\n"
+                   "int longbitfield : 6;",
+                   Alignment));
+  EXPECT_EQ("int a            : 5;\n"
+            "int one          : 1;\n"
+            "// line comment\n"
+            "int longbitfield : 6;",
+            format("int a : 5;\n"
+                   "int one : 1;\n"
+                   "// line comment\n"
+                   "int longbitfield : 6;",
+                   Alignment));
+
+  /* Test across comments and newlines */
+  EXPECT_EQ("int a            : 5;\n"
+            "/* block comment */\n"
+            "\n"
+            "int longbitfield : 6;",
+            format("int a : 5;\n"
+                   "/* block comment */\n"
+                   "\n"
+                   "int longbitfield : 6;",
+                   Alignment));
+  EXPECT_EQ("int a            : 5;\n"
+            "int one          : 1;\n"
+            "\n"
+            "// line comment\n"
+            "\n"
+            "int longbitfield : 6;",
+            format("int a : 5;\n"
+                   "int one : 1;\n"
+                   "\n"
+                   "// line comment \n"
+                   "\n"
+                   "int longbitfield : 6;",
+                   Alignment));
+}
+
+TEST_F(FormatTest, AlignConsecutiveAssignmentsAcrossComments) {
+  FormatStyle Alignment = getLLVMStyle();
+  Alignment.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_AcrossComments;
+
+  Alignment.MaxEmptyLinesToKeep = 10;
+  /* Test alignment across empty lines */
+  EXPECT_EQ("int a = 5;\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a       = 5;\n"
+                   "\n"
+                   "int oneTwoThree= 123;",
+                   Alignment));
+  EXPECT_EQ("int a   = 5;\n"
+            "int one = 1;\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "int one = 1;\n"
+                   "\n"
+                   "int oneTwoThree = 123;",
+                   Alignment));
+
+  /* Test across comments */
+  EXPECT_EQ("int a           = 5;\n"
+            "/* block comment */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "// line comment\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "/*\n"
+            " * multi-line block comment\n"
+            " */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "/*\n"
+                   " * multi-line block comment\n"
+                   " */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "//\n"
+            "// multi-line line comment\n"
+            "//\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "//\n"
+                   "// multi-line line comment\n"
+                   "//\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  /* Test across comments and newlines */
+  EXPECT_EQ("int a = 5;\n"
+            "\n"
+            "/* block comment */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a = 5;\n"
+            "\n"
+            "// line comment\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+}
+
+TEST_F(FormatTest, AlignConsecutiveAssignmentsAcrossEmptyLinesAndComments) {
+  FormatStyle Alignment = getLLVMStyle();
+  Alignment.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
+  Alignment.AlignConsecutiveAssignments =
+      FormatStyle::ACS_AcrossEmptyLinesAndComments;
+  verifyFormat("int a           = 5;\n"
+               "int oneTwoThree = 123;",
+               Alignment);
+  verifyFormat("int a           = method();\n"
+               "int oneTwoThree = 133;",
+               Alignment);
+  verifyFormat("a &= 5;\n"
+               "bcd *= 5;\n"
+               "ghtyf += 5;\n"
+               "dvfvdb -= 5;\n"
+               "a /= 5;\n"
+               "vdsvsv %= 5;\n"
+               "sfdbddfbdfbb ^= 5;\n"
+               "dvsdsv |= 5;\n"
+               "int dsvvdvsdvvv = 123;",
+               Alignment);
+  verifyFormat("int i = 1, j = 10;\n"
+               "something = 2000;",
+               Alignment);
+  verifyFormat("something = 2000;\n"
+               "int i = 1, j = 10;\n",
+               Alignment);
+  verifyFormat("something = 2000;\n"
+               "another   = 911;\n"
+               "int i = 1, j = 10;\n"
+               "oneMore = 1;\n"
+               "i       = 2;",
+               Alignment);
+  verifyFormat("int a   = 5;\n"
+               "int one = 1;\n"
+               "method();\n"
+               "int oneTwoThree = 123;\n"
+               "int oneTwo      = 12;",
+               Alignment);
+  verifyFormat("int oneTwoThree = 123;\n"
+               "int oneTwo      = 12;\n"
+               "method();\n",
+               Alignment);
+  verifyFormat("int oneTwoThree = 123; // comment\n"
+               "int oneTwo      = 12;  // comment",
+               Alignment);
+
+  // Bug 25167
+  /* Uncomment when fixed
+    verifyFormat("#if A\n"
+                 "#else\n"
+                 "int aaaaaaaa = 12;\n"
+                 "#endif\n"
+                 "#if B\n"
+                 "#else\n"
+                 "int a = 12;\n"
+                 "#endif\n",
+                 Alignment);
+    verifyFormat("enum foo {\n"
+                 "#if A\n"
+                 "#else\n"
+                 "  aaaaaaaa = 12;\n"
+                 "#endif\n"
+                 "#if B\n"
+                 "#else\n"
+                 "  a = 12;\n"
+                 "#endif\n"
+                 "};\n",
+                 Alignment);
+  */
+
+  Alignment.MaxEmptyLinesToKeep = 10;
+  /* Test alignment across empty lines */
+  EXPECT_EQ("int a           = 5;\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a       = 5;\n"
+                   "\n"
+                   "int oneTwoThree= 123;",
+                   Alignment));
+  EXPECT_EQ("int a           = 5;\n"
+            "int one         = 1;\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "int one = 1;\n"
+                   "\n"
+                   "int oneTwoThree = 123;",
+                   Alignment));
+  EXPECT_EQ("int a           = 5;\n"
+            "int one         = 1;\n"
+            "\n"
+            "int oneTwoThree = 123;\n"
+            "int oneTwo      = 12;",
+            format("int a = 5;\n"
+                   "int one = 1;\n"
+                   "\n"
+                   "int oneTwoThree = 123;\n"
+                   "int oneTwo = 12;",
+                   Alignment));
+
+  /* Test across comments */
+  EXPECT_EQ("int a           = 5;\n"
+            "/* block comment */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "// line comment\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  /* Test across comments and newlines */
+  EXPECT_EQ("int a           = 5;\n"
+            "\n"
+            "/* block comment */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "/* block comment */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "\n"
+            "// line comment\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "// line comment\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "//\n"
+            "// multi-line line comment\n"
+            "//\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "//\n"
+                   "// multi-line line comment\n"
+                   "//\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "/*\n"
+            " *  multi-line block comment\n"
+            " */\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "/*\n"
+                   " *  multi-line block comment\n"
+                   " */\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "\n"
+            "/* block comment */\n"
+            "\n"
+            "\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "/* block comment */\n"
+                   "\n"
+                   "\n"
+                   "\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  EXPECT_EQ("int a           = 5;\n"
+            "\n"
+            "// line comment\n"
+            "\n"
+            "\n"
+            "\n"
+            "int oneTwoThree = 123;",
+            format("int a = 5;\n"
+                   "\n"
+                   "// line comment\n"
+                   "\n"
+                   "\n"
+                   "\n"
+                   "int oneTwoThree=123;",
+                   Alignment));
+
+  Alignment.AlignEscapedNewlines = FormatStyle::ENAS_DontAlign;
+  verifyFormat("#define A \\\n"
+               "  int aaaa       = 12; \\\n"
+               "  int b          = 23; \\\n"
+               "  int ccc        = 234; \\\n"
+               "  int dddddddddd = 2345;",
+               Alignment);
+  Alignment.AlignEscapedNewlines = FormatStyle::ENAS_Left;
+  verifyFormat("#define A               \\\n"
+               "  int aaaa       = 12;  \\\n"
+               "  int b          = 23;  \\\n"
+               "  int ccc        = 234; \\\n"
+               "  int dddddddddd = 2345;",
+               Alignment);
+  Alignment.AlignEscapedNewlines = FormatStyle::ENAS_Right;
+  verifyFormat("#define A                                                      "
+               "                \\\n"
+               "  int aaaa       = 12;                                         "
+               "                \\\n"
+               "  int b          = 23;                                         "
+               "                \\\n"
+               "  int ccc        = 234;                                        "
+               "                \\\n"
+               "  int dddddddddd = 2345;",
+               Alignment);
+  verifyFormat("void SomeFunction(int parameter = 1, int i = 2, int j = 3, int "
+               "k = 4, int l = 5,\n"
+               "                  int m = 6) {\n"
+               "  int j      = 10;\n"
+               "  otherThing = 1;\n"
+               "}",
+               Alignment);
+  verifyFormat("void SomeFunction(int parameter = 0) {\n"
+               "  int i   = 1;\n"
+               "  int j   = 2;\n"
+               "  int big = 10000;\n"
+               "}",
+               Alignment);
+  verifyFormat("class C {\n"
+               "public:\n"
+               "  int i            = 1;\n"
+               "  virtual void f() = 0;\n"
+               "};",
+               Alignment);
+  verifyFormat("int i = 1;\n"
+               "if (SomeType t = getSomething()) {\n"
+               "}\n"
+               "int j   = 2;\n"
+               "int big = 10000;",
+               Alignment);
+  verifyFormat("int j = 7;\n"
+               "for (int k = 0; k < N; ++k) {\n"
+               "}\n"
+               "int j   = 2;\n"
+               "int big = 10000;\n"
+               "}",
+               Alignment);
+  Alignment.BreakBeforeBinaryOperators = FormatStyle::BOS_All;
+  verifyFormat("int i = 1;\n"
+               "LooooooooooongType loooooooooooooooooooooongVariable\n"
+               "    = someLooooooooooooooooongFunction();\n"
+               "int j = 2;",
+               Alignment);
+  Alignment.BreakBeforeBinaryOperators = FormatStyle::BOS_None;
+  verifyFormat("int i = 1;\n"
+               "LooooooooooongType loooooooooooooooooooooongVariable =\n"
+               "    someLooooooooooooooooongFunction();\n"
+               "int j = 2;",
+               Alignment);
+
+  verifyFormat("auto lambda = []() {\n"
+               "  auto i = 0;\n"
+               "  return 0;\n"
+               "};\n"
+               "int i  = 0;\n"
+               "auto v = type{\n"
+               "    i = 1,   //\n"
+               "    (i = 2), //\n"
+               "    i = 3    //\n"
+               "};",
+               Alignment);
+
+  verifyFormat(
+      "int i      = 1;\n"
+      "SomeType a = SomeFunction(looooooooooooooooooooooongParameterA,\n"
+      "                          loooooooooooooooooooooongParameterB);\n"
+      "int j      = 2;",
+      Alignment);
+
+  verifyFormat("template <typename T, typename T_0 = very_long_type_name_0,\n"
+               "          typename B   = very_long_type_name_1,\n"
+               "          typename T_2 = very_long_type_name_2>\n"
+               "auto foo() {}\n",
+               Alignment);
+  verifyFormat("int a, b = 1;\n"
+               "int c  = 2;\n"
+               "int dd = 3;\n",
+               Alignment);
+  verifyFormat("int aa       = ((1 > 2) ? 3 : 4);\n"
+               "float b[1][] = {{3.f}};\n",
+               Alignment);
+  verifyFormat("for (int i = 0; i < 1; i++)\n"
+               "  int x = 1;\n",
+               Alignment);
+  verifyFormat("for (i = 0; i < 1; i++)\n"
+               "  x = 1;\n"
+               "y = 1;\n",
+               Alignment);
+
+  Alignment.ReflowComments = true;
+  Alignment.ColumnLimit = 50;
+  EXPECT_EQ("int x   = 0;\n"
+            "int yy  = 1; /// specificlennospace\n"
+            "int zzz = 2;\n",
+            format("int x   = 0;\n"
+                   "int yy  = 1; ///specificlennospace\n"
+                   "int zzz = 2;\n",
+                   Alignment));
 }
 
 TEST_F(FormatTest, AlignConsecutiveAssignments) {
   FormatStyle Alignment = getLLVMStyle();
-  Alignment.AlignConsecutiveMacros = true;
-  Alignment.AlignConsecutiveAssignments = false;
+  Alignment.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_None;
   verifyFormat("int a = 5;\n"
                "int oneTwoThree = 123;",
                Alignment);
@@ -12574,7 +13328,7 @@ TEST_F(FormatTest, AlignConsecutiveAssignments) {
                "int oneTwoThree = 123;",
                Alignment);
 
-  Alignment.AlignConsecutiveAssignments = true;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   verifyFormat("int a           = 5;\n"
                "int oneTwoThree = 123;",
                Alignment);
@@ -12790,7 +13544,7 @@ TEST_F(FormatTest, AlignConsecutiveAssignments) {
 
 TEST_F(FormatTest, AlignConsecutiveBitFields) {
   FormatStyle Alignment = getLLVMStyle();
-  Alignment.AlignConsecutiveBitFields = true;
+  Alignment.AlignConsecutiveBitFields = FormatStyle::ACS_Consecutive;
   verifyFormat("int const a     : 5;\n"
                "int oneTwoThree : 23;",
                Alignment);
@@ -12800,7 +13554,7 @@ TEST_F(FormatTest, AlignConsecutiveBitFields) {
                "int oneTwoThree : 23 = 0;",
                Alignment);
 
-  Alignment.AlignConsecutiveDeclarations = true;
+  Alignment.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   verifyFormat("int const a           : 5;\n"
                "int       oneTwoThree : 23;",
                Alignment);
@@ -12813,7 +13567,7 @@ TEST_F(FormatTest, AlignConsecutiveBitFields) {
                "int       oneTwoThree : 23 = 0;",
                Alignment);
 
-  Alignment.AlignConsecutiveAssignments = true;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   verifyFormat("int const a           : 5  = 1;\n"
                "int       oneTwoThree : 23 = 0;",
                Alignment);
@@ -12847,8 +13601,8 @@ TEST_F(FormatTest, AlignConsecutiveBitFields) {
 
 TEST_F(FormatTest, AlignConsecutiveDeclarations) {
   FormatStyle Alignment = getLLVMStyle();
-  Alignment.AlignConsecutiveMacros = true;
-  Alignment.AlignConsecutiveDeclarations = false;
+  Alignment.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
+  Alignment.AlignConsecutiveDeclarations = FormatStyle::ACS_None;
   verifyFormat("float const a = 5;\n"
                "int oneTwoThree = 123;",
                Alignment);
@@ -12856,7 +13610,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                "float const oneTwoThree = 123;",
                Alignment);
 
-  Alignment.AlignConsecutiveDeclarations = true;
+  Alignment.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   verifyFormat("float const a = 5;\n"
                "int         oneTwoThree = 123;",
                Alignment);
@@ -12953,7 +13707,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
   verifyFormat("int    a(int x, void (*fp)(int y));\n"
                "double b();",
                Alignment);
-  Alignment.AlignConsecutiveAssignments = true;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   // Ensure recursive alignment is broken by function braces, so that the
   // "a = 1" does not align with subsequent assignments inside the function
   // body.
@@ -13016,7 +13770,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                    "int ll=10000;\n"
                    "}",
                    Alignment));
-  Alignment.AlignConsecutiveAssignments = false;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_None;
   Alignment.AlignEscapedNewlines = FormatStyle::ENAS_DontAlign;
   verifyFormat("#define A \\\n"
                "  int       aaaa = 12; \\\n"
@@ -13085,7 +13839,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                "int j = 2;",
                Alignment);
 
-  Alignment.AlignConsecutiveAssignments = true;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   verifyFormat("auto lambda = []() {\n"
                "  auto  ii = 0;\n"
                "  float j  = 0;\n"
@@ -13099,7 +13853,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                "    i = 3    //\n"
                "};",
                Alignment);
-  Alignment.AlignConsecutiveAssignments = false;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_None;
 
   verifyFormat(
       "int      i = 1;\n"
@@ -13112,7 +13866,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
   // We expect declarations and assignments to align, as long as it doesn't
   // exceed the column limit, starting a new alignment sequence whenever it
   // happens.
-  Alignment.AlignConsecutiveAssignments = true;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   Alignment.ColumnLimit = 30;
   verifyFormat("float    ii              = 1;\n"
                "unsigned j               = 2;\n"
@@ -13122,7 +13876,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                "int              myvar = 1;",
                Alignment);
   Alignment.ColumnLimit = 80;
-  Alignment.AlignConsecutiveAssignments = false;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_None;
 
   verifyFormat(
       "template <typename LongTemplate, typename VeryLongTemplateTypeName,\n"
@@ -13136,7 +13890,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
   verifyFormat("int   aa = ((1 > 2) ? 3 : 4);\n"
                "float b[1][] = {{3.f}};\n",
                Alignment);
-  Alignment.AlignConsecutiveAssignments = true;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   verifyFormat("float a, b = 1;\n"
                "int   c  = 2;\n"
                "int   dd = 3;\n",
@@ -13144,7 +13898,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
   verifyFormat("int   aa     = ((1 > 2) ? 3 : 4);\n"
                "float b[1][] = {{3.f}};\n",
                Alignment);
-  Alignment.AlignConsecutiveAssignments = false;
+  Alignment.AlignConsecutiveAssignments = FormatStyle::ACS_None;
 
   Alignment.ColumnLimit = 30;
   Alignment.BinPackParameters = false;
@@ -13175,7 +13929,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
   Alignment.PointerAlignment = FormatStyle::PAS_Right;
 
   // See llvm.org/PR35641
-  Alignment.AlignConsecutiveDeclarations = true;
+  Alignment.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   verifyFormat("int func() { //\n"
                "  int      b;\n"
                "  unsigned c;\n"
@@ -13184,7 +13938,7 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
 
   // See PR37175
   FormatStyle Style = getMozillaStyle();
-  Style.AlignConsecutiveDeclarations = true;
+  Style.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   EXPECT_EQ("DECOR1 /**/ int8_t /**/ DECOR2 /**/\n"
             "foo(int a);",
             format("DECOR1 /**/ int8_t /**/ DECOR2 /**/ foo (int a);", Style));
@@ -14374,10 +15128,6 @@ TEST_F(FormatTest, ParsesConfigurationBools) {
   FormatStyle Style = {};
   Style.Language = FormatStyle::LK_Cpp;
   CHECK_PARSE_BOOL(AlignTrailingComments);
-  CHECK_PARSE_BOOL(AlignConsecutiveAssignments);
-  CHECK_PARSE_BOOL(AlignConsecutiveBitFields);
-  CHECK_PARSE_BOOL(AlignConsecutiveDeclarations);
-  CHECK_PARSE_BOOL(AlignConsecutiveMacros);
   CHECK_PARSE_BOOL(AllowAllArgumentsOnNextLine);
   CHECK_PARSE_BOOL(AllowAllConstructorInitializersOnNextLine);
   CHECK_PARSE_BOOL(AllowAllParametersOfDeclarationOnNextLine);
@@ -14472,6 +15222,70 @@ TEST_F(FormatTest, ParsesConfiguration) {
   CHECK_PARSE("ContinuationIndentWidth: 11", ContinuationIndentWidth, 11u);
   CHECK_PARSE("CommentPragmas: '// abc$'", CommentPragmas, "// abc$");
 
+  Style.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
+  CHECK_PARSE("AlignConsecutiveAssignments: None", AlignConsecutiveAssignments,
+              FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveAssignments: Consecutive",
+              AlignConsecutiveAssignments, FormatStyle::ACS_Consecutive);
+  CHECK_PARSE("AlignConsecutiveAssignments: AcrossEmptyLines",
+              AlignConsecutiveAssignments, FormatStyle::ACS_AcrossEmptyLines);
+  CHECK_PARSE("AlignConsecutiveAssignments: AcrossEmptyLinesAndComments",
+              AlignConsecutiveAssignments,
+              FormatStyle::ACS_AcrossEmptyLinesAndComments);
+  // For backwards compability, false / true should still parse
+  CHECK_PARSE("AlignConsecutiveAssignments: false", AlignConsecutiveAssignments,
+              FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveAssignments: true", AlignConsecutiveAssignments,
+              FormatStyle::ACS_Consecutive);
+
+  Style.AlignConsecutiveBitFields = FormatStyle::ACS_Consecutive;
+  CHECK_PARSE("AlignConsecutiveBitFields: None", AlignConsecutiveBitFields,
+              FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveBitFields: Consecutive",
+              AlignConsecutiveBitFields, FormatStyle::ACS_Consecutive);
+  CHECK_PARSE("AlignConsecutiveBitFields: AcrossEmptyLines",
+              AlignConsecutiveBitFields, FormatStyle::ACS_AcrossEmptyLines);
+  CHECK_PARSE("AlignConsecutiveBitFields: AcrossEmptyLinesAndComments",
+              AlignConsecutiveBitFields,
+              FormatStyle::ACS_AcrossEmptyLinesAndComments);
+  // For backwards compability, false / true should still parse
+  CHECK_PARSE("AlignConsecutiveBitFields: false", AlignConsecutiveBitFields,
+              FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveBitFields: true", AlignConsecutiveBitFields,
+              FormatStyle::ACS_Consecutive);
+
+  Style.AlignConsecutiveMacros = FormatStyle::ACS_Consecutive;
+  CHECK_PARSE("AlignConsecutiveMacros: None", AlignConsecutiveMacros,
+              FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveMacros: Consecutive", AlignConsecutiveMacros,
+              FormatStyle::ACS_Consecutive);
+  CHECK_PARSE("AlignConsecutiveMacros: AcrossEmptyLines",
+              AlignConsecutiveMacros, FormatStyle::ACS_AcrossEmptyLines);
+  CHECK_PARSE("AlignConsecutiveMacros: AcrossEmptyLinesAndComments",
+              AlignConsecutiveMacros,
+              FormatStyle::ACS_AcrossEmptyLinesAndComments);
+  // For backwards compability, false / true should still parse
+  CHECK_PARSE("AlignConsecutiveMacros: false", AlignConsecutiveMacros,
+              FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveMacros: true", AlignConsecutiveMacros,
+              FormatStyle::ACS_Consecutive);
+
+  Style.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
+  CHECK_PARSE("AlignConsecutiveDeclarations: None",
+              AlignConsecutiveDeclarations, FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveDeclarations: Consecutive",
+              AlignConsecutiveDeclarations, FormatStyle::ACS_Consecutive);
+  CHECK_PARSE("AlignConsecutiveDeclarations: AcrossEmptyLines",
+              AlignConsecutiveDeclarations, FormatStyle::ACS_AcrossEmptyLines);
+  CHECK_PARSE("AlignConsecutiveDeclarations: AcrossEmptyLinesAndComments",
+              AlignConsecutiveDeclarations,
+              FormatStyle::ACS_AcrossEmptyLinesAndComments);
+  // For backwards compability, false / true should still parse
+  CHECK_PARSE("AlignConsecutiveDeclarations: false",
+              AlignConsecutiveDeclarations, FormatStyle::ACS_None);
+  CHECK_PARSE("AlignConsecutiveDeclarations: true",
+              AlignConsecutiveDeclarations, FormatStyle::ACS_Consecutive);
+
   Style.PointerAlignment = FormatStyle::PAS_Middle;
   CHECK_PARSE("PointerAlignment: Left", PointerAlignment,
               FormatStyle::PAS_Left);
@@ -17400,7 +18214,7 @@ TEST_F(FormatTest, WhitespaceSensitiveMacros) {
             format("FOO(String-ized&Messy+But,: :\n"
                    "       Still=Intentional);",
                    Style));
-  Style.AlignConsecutiveAssignments = true;
+  Style.AlignConsecutiveAssignments = FormatStyle::ACS_Consecutive;
   EXPECT_EQ("FOO(String-ized=&Messy+But,: :\n"
             "       Still=Intentional);",
             format("FOO(String-ized=&Messy+But,: :\n"
@@ -17871,7 +18685,7 @@ TEST_F(FormatTest, StatementAttributeLikeMacros) {
 
   EXPECT_EQ(Source, format(Source, Style));
 
-  Style.AlignConsecutiveDeclarations = true;
+  Style.AlignConsecutiveDeclarations = FormatStyle::ACS_Consecutive;
   EXPECT_EQ("void Foo::slot() {\n"
             "  unsigned char MyChar = 'x';\n"
             "  emit          signal(MyChar);\n"
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index cd0faf215a5a..b34b47a18b85 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -191,8 +191,7 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &res,
 
   // Parse the arguments
   const llvm::opt::OptTable &opts = clang::driver::getDriverOptTable();
-  const unsigned includedFlagsBitmask =
-      clang::driver::options::FC1Option;
+  const unsigned includedFlagsBitmask = clang::driver::options::FC1Option;
   unsigned missingArgIndex, missingArgCount;
   llvm::opt::InputArgList args = opts.ParseArgs(
       commandLineArgs, missingArgIndex, missingArgCount, includedFlagsBitmask);
diff --git a/flang/test/Preprocessing/assert.F90 b/flang/test/Preprocessing/assert.F90
index 64d49e6f1cbd..5d4837efd6d5 100644
--- a/flang/test/Preprocessing/assert.F90
+++ b/flang/test/Preprocessing/assert.F90
@@ -1,4 +1,4 @@
-!RUN: %f18 -E %s 2>&1 | FileCheck %s
+!RUN: %flang -E %s 2>&1 | FileCheck %s
 !CHECK: if(.not.(.true.)) error stop "assert(" // ".TRUE." // ") failed " // "at ""
 !CHECK-SAME: assert.F90"": " // "7"
 #define STR(x) #x
diff --git a/flang/test/Preprocessing/compiler_defined_macros.F90 b/flang/test/Preprocessing/compiler_defined_macros.F90
index ba20f6d39622..f5f955bbc750 100644
--- a/flang/test/Preprocessing/compiler_defined_macros.F90
+++ b/flang/test/Preprocessing/compiler_defined_macros.F90
@@ -3,7 +3,7 @@
 !CHECK: flang_major = {{[1-9][0-9]*$}}
 !CHECK: flang_minor = {{[0-9]+$}}
 !CHECK: flang_patchlevel = {{[0-9]+$}}
-!RUN: %f18 -E %s | FileCheck  --ignore-case %s
+!RUN: %flang -E %s | FileCheck  --ignore-case %s
 
   
 integer, parameter :: flang_major = __flang_major__
diff --git a/flang/test/Preprocessing/defines.F90 b/flang/test/Preprocessing/defines.F90
index 449c54cbb01f..d59789f17f6d 100644
--- a/flang/test/Preprocessing/defines.F90
+++ b/flang/test/Preprocessing/defines.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E -DFOO=1 -DBAR=2 %s | FileCheck %s
+! RUN: %flang -E -DFOO=1 -DBAR=2 %s | FileCheck %s
 
 ! CHECK: integer :: a = 1
   integer :: a = FOO
diff --git a/flang/test/Preprocessing/fixed-rescan.F b/flang/test/Preprocessing/fixed-rescan.F
index 3d6ba9a8c6f4..ac983e7c3194 100644
--- a/flang/test/Preprocessing/fixed-rescan.F
+++ b/flang/test/Preprocessing/fixed-rescan.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s | FileCheck %s
+! RUN: %flang -E %s | FileCheck %s
 ! CHECK: callbar
 ! Ensure that rescanned lines after macro replacement are not
 ! misinterpreted as fixed-form comments when they start with C or D.
diff --git a/flang/test/Preprocessing/hollerith.f b/flang/test/Preprocessing/hollerith.f
index 0c920e0678e1..5f20259c57a8 100644
--- a/flang/test/Preprocessing/hollerith.f
+++ b/flang/test/Preprocessing/hollerith.f
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: character*1hi
 ! CHECK: dataa/1*1h /
 ! CHECK: datab/1*1h /
diff --git a/flang/test/Preprocessing/include-comment.F90 b/flang/test/Preprocessing/include-comment.F90
index 6ac475f76e46..239d9b7cc2e3 100644
--- a/flang/test/Preprocessing/include-comment.F90
+++ b/flang/test/Preprocessing/include-comment.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -I%S -E %s 2>&1 | FileCheck %s
+! RUN: %flang -I%S -E %s 2>&1 | FileCheck %s
 ! CHECK-NOT: :3:
 #include <empty.h> ! comment
 ! CHECK-NOT: :5:
diff --git a/flang/test/Preprocessing/pp001.F b/flang/test/Preprocessing/pp001.F
index 95198641655a..455da7302763 100644
--- a/flang/test/Preprocessing/pp001.F
+++ b/flang/test/Preprocessing/pp001.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 * keyword macros
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp002.F b/flang/test/Preprocessing/pp002.F
index b4cbd7545adf..d673fcdbd6da 100644
--- a/flang/test/Preprocessing/pp002.F
+++ b/flang/test/Preprocessing/pp002.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm.eq.777)then
 * #undef
       integer, parameter :: KWM = 777
diff --git a/flang/test/Preprocessing/pp003.F b/flang/test/Preprocessing/pp003.F
index 4df10c47d235..2062dbcee703 100644
--- a/flang/test/Preprocessing/pp003.F
+++ b/flang/test/Preprocessing/pp003.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * function-like macros
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp004.F b/flang/test/Preprocessing/pp004.F
index 788d3682ab98..f5724df45d52 100644
--- a/flang/test/Preprocessing/pp004.F
+++ b/flang/test/Preprocessing/pp004.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm.eq.777)then
 * KWMs case-sensitive
       integer, parameter :: KWM = 777
diff --git a/flang/test/Preprocessing/pp005.F b/flang/test/Preprocessing/pp005.F
index 390e662e3557..48cf12df628e 100644
--- a/flang/test/Preprocessing/pp005.F
+++ b/flang/test/Preprocessing/pp005.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=777
 * KWM split across continuation, implicit padding
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp006.F b/flang/test/Preprocessing/pp006.F
index 3c44728f56b5..f878bfddd143 100644
--- a/flang/test/Preprocessing/pp006.F
+++ b/flang/test/Preprocessing/pp006.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=777
 * ditto, but with intervening *comment line
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp007.F b/flang/test/Preprocessing/pp007.F
index ca78f4f4c6c5..a5d871a2d1c1 100644
--- a/flang/test/Preprocessing/pp007.F
+++ b/flang/test/Preprocessing/pp007.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=kwm
 * KWM split across continuation, clipped after column 72
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp008.F b/flang/test/Preprocessing/pp008.F
index 9dd7bd92ea50..7e580efddb65 100644
--- a/flang/test/Preprocessing/pp008.F
+++ b/flang/test/Preprocessing/pp008.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=kwm
 * KWM with spaces in name at invocation NOT replaced
       integer, parameter :: KWM = 777
diff --git a/flang/test/Preprocessing/pp009.F b/flang/test/Preprocessing/pp009.F
index c29021da484a..b6a6249c0544 100644
--- a/flang/test/Preprocessing/pp009.F
+++ b/flang/test/Preprocessing/pp009.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call split across continuation, implicit padding
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp010.F b/flang/test/Preprocessing/pp010.F
index 4a812cde86af..e860bd9f3abe 100644
--- a/flang/test/Preprocessing/pp010.F
+++ b/flang/test/Preprocessing/pp010.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * ditto, but with intervening *comment line
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp011.F b/flang/test/Preprocessing/pp011.F
index c106c8bc3f5c..af0e0e88b333 100644
--- a/flang/test/Preprocessing/pp011.F
+++ b/flang/test/Preprocessing/pp011.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=iflm(666)
 * FLM call name split across continuation, clipped
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp012.F b/flang/test/Preprocessing/pp012.F
index 411cfb887bb1..12ab61233275 100644
--- a/flang/test/Preprocessing/pp012.F
+++ b/flang/test/Preprocessing/pp012.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call name split across continuation
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp013.F b/flang/test/Preprocessing/pp013.F
index f05e2e30fee6..07bb25e7c8d6 100644
--- a/flang/test/Preprocessing/pp013.F
+++ b/flang/test/Preprocessing/pp013.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call split between name and (
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp014.F b/flang/test/Preprocessing/pp014.F
index 470966096e4e..7ac5d382bc7d 100644
--- a/flang/test/Preprocessing/pp014.F
+++ b/flang/test/Preprocessing/pp014.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call split between name and (, with intervening *comment
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp015.F b/flang/test/Preprocessing/pp015.F
index 8320bd704fed..700da10a743e 100644
--- a/flang/test/Preprocessing/pp015.F
+++ b/flang/test/Preprocessing/pp015.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call split between name and (, clipped
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp016.F b/flang/test/Preprocessing/pp016.F
index 95c89f8e1a03..c5d90e7e2ffb 100644
--- a/flang/test/Preprocessing/pp016.F
+++ b/flang/test/Preprocessing/pp016.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call split between name and ( and in argument
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp017.F b/flang/test/Preprocessing/pp017.F
index a11f18514783..49fff58cdbb4 100644
--- a/flang/test/Preprocessing/pp017.F
+++ b/flang/test/Preprocessing/pp017.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 * KLM rescan
       integer, parameter :: KWM = 666, KWM2 = 667
diff --git a/flang/test/Preprocessing/pp018.F b/flang/test/Preprocessing/pp018.F
index 69c24e59f53d..d6a42160404c 100644
--- a/flang/test/Preprocessing/pp018.F
+++ b/flang/test/Preprocessing/pp018.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm2.eq.777)then
 * KLM rescan with #undef (so rescan is after expansion)
       integer, parameter :: KWM2 = 777, KWM = 667
diff --git a/flang/test/Preprocessing/pp019.F b/flang/test/Preprocessing/pp019.F
index d607ad348407..a8c0ff6b598e 100644
--- a/flang/test/Preprocessing/pp019.F
+++ b/flang/test/Preprocessing/pp019.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM rescan
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp020.F b/flang/test/Preprocessing/pp020.F
index 88525003ba89..9fe9630a81c2 100644
--- a/flang/test/Preprocessing/pp020.F
+++ b/flang/test/Preprocessing/pp020.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((111)+666)
 * FLM expansion of argument
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp021.F b/flang/test/Preprocessing/pp021.F
index 1662a680f5aa..2936264f6c9c 100644
--- a/flang/test/Preprocessing/pp021.F
+++ b/flang/test/Preprocessing/pp021.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: ch='KWM'
 ! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in 'literal'
diff --git a/flang/test/Preprocessing/pp022.F b/flang/test/Preprocessing/pp022.F
index 026c02e9d4f0..647b683ec5bc 100644
--- a/flang/test/Preprocessing/pp022.F
+++ b/flang/test/Preprocessing/pp022.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: ch="KWM"
 ! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in "literal"
diff --git a/flang/test/Preprocessing/pp023.F b/flang/test/Preprocessing/pp023.F
index a0d053f6addc..ecad31b57b6a 100644
--- a/flang/test/Preprocessing/pp023.F
+++ b/flang/test/Preprocessing/pp023.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: ch=3hKWM
 ! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in 9HHOLLERITH literal
diff --git a/flang/test/Preprocessing/pp024.F b/flang/test/Preprocessing/pp024.F
index 6ea76cc9c24d..7f8e3bb99083 100644
--- a/flang/test/Preprocessing/pp024.F
+++ b/flang/test/Preprocessing/pp024.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: 100format(3hKWM)
 ! CHECK: if(ch.eq.'KWM')then
 * KWM NOT expanded in Hollerith in FORMAT
diff --git a/flang/test/Preprocessing/pp025.F b/flang/test/Preprocessing/pp025.F
index 49521d443bd3..bb7bc212776b 100644
--- a/flang/test/Preprocessing/pp025.F
+++ b/flang/test/Preprocessing/pp025.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=ikwm2z
 * KWM expansion is before token pasting due to fixed-form space removal
       integer, parameter :: IKWM2Z = 777
diff --git a/flang/test/Preprocessing/pp026.F b/flang/test/Preprocessing/pp026.F
index b551f3b173ed..e37700129e9c 100644
--- a/flang/test/Preprocessing/pp026.F
+++ b/flang/test/Preprocessing/pp026.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((111)+666)
 * ## token pasting works in FLM
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp027.F b/flang/test/Preprocessing/pp027.F
index c1b787f9366d..577bd48b0da3 100644
--- a/flang/test/Preprocessing/pp027.F
+++ b/flang/test/Preprocessing/pp027.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: kwm=666
 ! CHECK: if(777.eq.777)then
 * #DEFINE works in fixed form
diff --git a/flang/test/Preprocessing/pp028.F b/flang/test/Preprocessing/pp028.F
index 51eab84f3687..3530628d5c50 100644
--- a/flang/test/Preprocessing/pp028.F
+++ b/flang/test/Preprocessing/pp028.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=kw
 * fixed-form clipping done before KWM expansion on source line
       integer, parameter :: KW = 777
diff --git a/flang/test/Preprocessing/pp029.F b/flang/test/Preprocessing/pp029.F
index 9be309f75143..23c0d337ad92 100644
--- a/flang/test/Preprocessing/pp029.F
+++ b/flang/test/Preprocessing/pp029.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 * \ newline allowed in #define
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp030.F b/flang/test/Preprocessing/pp030.F
index c04cf949f414..91b12178281d 100644
--- a/flang/test/Preprocessing/pp030.F
+++ b/flang/test/Preprocessing/pp030.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 * /* C comment */ erased from #define
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp031.F b/flang/test/Preprocessing/pp031.F
index 90b14647c4c9..d8286169c02f 100644
--- a/flang/test/Preprocessing/pp031.F
+++ b/flang/test/Preprocessing/pp031.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777//ccomment.eq.777)then
 ! CHECK: print*,'pp031.F no: ',777//ccomment
 *  // C++ comment NOT erased from #define
diff --git a/flang/test/Preprocessing/pp032.F b/flang/test/Preprocessing/pp032.F
index 6b779141ecb5..22350e7349f6 100644
--- a/flang/test/Preprocessing/pp032.F
+++ b/flang/test/Preprocessing/pp032.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 ! CHECK: print*,'pp032.F no: ',777
 * /* C comment */ \ newline erased from #define
diff --git a/flang/test/Preprocessing/pp033.F b/flang/test/Preprocessing/pp033.F
index 3364527cba6a..1b32ae6d90ca 100644
--- a/flang/test/Preprocessing/pp033.F
+++ b/flang/test/Preprocessing/pp033.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 ! CHECK: print*,'pp033.F no: ',777
 * /* C comment \ newline */ erased from #define
diff --git a/flang/test/Preprocessing/pp034.F b/flang/test/Preprocessing/pp034.F
index 0c64aca62b1d..d9f8a72ec74a 100644
--- a/flang/test/Preprocessing/pp034.F
+++ b/flang/test/Preprocessing/pp034.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 ! CHECK: print*,'pp034.F no: ',777
 * \ newline allowed in name on KWM definition
diff --git a/flang/test/Preprocessing/pp035.F b/flang/test/Preprocessing/pp035.F
index 808174129b89..0c0152a3df38 100644
--- a/flang/test/Preprocessing/pp035.F
+++ b/flang/test/Preprocessing/pp035.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777.eq.777)then
 ! CHECK: print*,'pp035.F no: ',777
 * #if 2 .LT. 3 works
diff --git a/flang/test/Preprocessing/pp036.F b/flang/test/Preprocessing/pp036.F
index b7024c41fa60..7894255dfb3b 100644
--- a/flang/test/Preprocessing/pp036.F
+++ b/flang/test/Preprocessing/pp036.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(.true.)then
 ! CHECK: print*,'pp036.F no: ',.true.
 * #define FALSE TRUE ...  .FALSE. -> .TRUE.
diff --git a/flang/test/Preprocessing/pp037.F b/flang/test/Preprocessing/pp037.F
index 52bfa8ffdab3..93c9b28e4d38 100644
--- a/flang/test/Preprocessing/pp037.F
+++ b/flang/test/Preprocessing/pp037.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(7777.eq.777)then
 ! CHECK: print*,'pp037.F no: ',7777
 * fixed-form clipping NOT applied to #define
diff --git a/flang/test/Preprocessing/pp038.F b/flang/test/Preprocessing/pp038.F
index 7386aeebddee..655551b7a9c6 100644
--- a/flang/test/Preprocessing/pp038.F
+++ b/flang/test/Preprocessing/pp038.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=((666)+111)
 * FLM call with closing ')' on next line (not a continuation)
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp039.F b/flang/test/Preprocessing/pp039.F
index 9124474b86ba..f80e0b074007 100644
--- a/flang/test/Preprocessing/pp039.F
+++ b/flang/test/Preprocessing/pp039.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res=iflm
 ! CHECK: (666)
 ! CHECK-NOT: res=((666)+111)
diff --git a/flang/test/Preprocessing/pp040.F b/flang/test/Preprocessing/pp040.F
index d589c38489cf..a3c2a6bd6b0a 100644
--- a/flang/test/Preprocessing/pp040.F
+++ b/flang/test/Preprocessing/pp040.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK-NOT: FAIL HARD!
 * #define KWM c, then KWM works as comment line initiator
 #define KWM c
diff --git a/flang/test/Preprocessing/pp041.F b/flang/test/Preprocessing/pp041.F
index dcf61536040b..9bc04db1e901 100644
--- a/flang/test/Preprocessing/pp041.F
+++ b/flang/test/Preprocessing/pp041.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: j=666wmj=j+1wm211
 * use KWM expansion as continuation indicators
 #define KWM 0
diff --git a/flang/test/Preprocessing/pp042.F b/flang/test/Preprocessing/pp042.F
index d5cc46a7a471..e5a679e226c8 100644
--- a/flang/test/Preprocessing/pp042.F
+++ b/flang/test/Preprocessing/pp042.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK-NOT: goto 2
 * #define c 1, then use c as label in fixed-form
 #define c 1
diff --git a/flang/test/Preprocessing/pp043.F b/flang/test/Preprocessing/pp043.F
index a079466e64f9..d8ed6b199bf1 100644
--- a/flang/test/Preprocessing/pp043.F
+++ b/flang/test/Preprocessing/pp043.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm.eq.777)then
 * #define with # in column 6 is a continuation line in fixed-form
       integer, parameter :: defineKWM666 = 555
diff --git a/flang/test/Preprocessing/pp044.F b/flang/test/Preprocessing/pp044.F
index c14b29c3050c..a699c08a2e4f 100644
--- a/flang/test/Preprocessing/pp044.F
+++ b/flang/test/Preprocessing/pp044.F
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK-NOT:z=111
 * #define directive amid continuations
       integer, parameter :: KWM = 222, KWM111 = 333, KWM222 = 555
diff --git a/flang/test/Preprocessing/pp101.F90 b/flang/test/Preprocessing/pp101.F90
index b0f860157c82..dbf2aa9bc30d 100644
--- a/flang/test/Preprocessing/pp101.F90
+++ b/flang/test/Preprocessing/pp101.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK:  if(777 .eq. 777) then
 ! keyword macros
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp102.F90 b/flang/test/Preprocessing/pp102.F90
index e2c3207a4c24..73d931ad1511 100644
--- a/flang/test/Preprocessing/pp102.F90
+++ b/flang/test/Preprocessing/pp102.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm .eq. 777) then
 ! #undef
       integer, parameter :: KWM = 777
diff --git a/flang/test/Preprocessing/pp103.F90 b/flang/test/Preprocessing/pp103.F90
index 3309d7f8205f..0384d8c193fa 100644
--- a/flang/test/Preprocessing/pp103.F90
+++ b/flang/test/Preprocessing/pp103.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((666)+111)
 ! function-like macros
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp104.F90 b/flang/test/Preprocessing/pp104.F90
index 51248c22154b..95653ff0f2aa 100644
--- a/flang/test/Preprocessing/pp104.F90
+++ b/flang/test/Preprocessing/pp104.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm .eq. 777) then
 ! KWMs case-sensitive
       integer, parameter :: KWM = 777
diff --git a/flang/test/Preprocessing/pp105.F90 b/flang/test/Preprocessing/pp105.F90
index 898a4a168137..b4f73da6fa24 100644
--- a/flang/test/Preprocessing/pp105.F90
+++ b/flang/test/Preprocessing/pp105.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = 777
 ! KWM call name split across continuation, with leading &
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp106.F90 b/flang/test/Preprocessing/pp106.F90
index d83085fc1d71..556d779048f6 100644
--- a/flang/test/Preprocessing/pp106.F90
+++ b/flang/test/Preprocessing/pp106.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = 777
 ! ditto, with & ! comment
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp107.F90 b/flang/test/Preprocessing/pp107.F90
index 6973127a4cfa..51aa750b8d23 100644
--- a/flang/test/Preprocessing/pp107.F90
+++ b/flang/test/Preprocessing/pp107.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = kwm
 ! KWM call name split across continuation, no leading &, with & ! comment
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp108.F90 b/flang/test/Preprocessing/pp108.F90
index b07ec984fd81..d891bf3db5e2 100644
--- a/flang/test/Preprocessing/pp108.F90
+++ b/flang/test/Preprocessing/pp108.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = kwm
 ! ditto, but without & ! comment
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp109.F90 b/flang/test/Preprocessing/pp109.F90
index e75fd10c1cb8..b488049c4032 100644
--- a/flang/test/Preprocessing/pp109.F90
+++ b/flang/test/Preprocessing/pp109.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((666)+111)
 ! FLM call name split with leading &
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp110.F90 b/flang/test/Preprocessing/pp110.F90
index 681408ebfddf..3f6180c45b13 100644
--- a/flang/test/Preprocessing/pp110.F90
+++ b/flang/test/Preprocessing/pp110.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((666)+111)
 ! ditto, with & ! comment
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp111.F90 b/flang/test/Preprocessing/pp111.F90
index 4b49bf6ad6c9..b0b1219cfca5 100644
--- a/flang/test/Preprocessing/pp111.F90
+++ b/flang/test/Preprocessing/pp111.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = iflm (666)
 ! FLM call name split across continuation, no leading &, with & ! comment
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp112.F90 b/flang/test/Preprocessing/pp112.F90
index 9828366bef73..1a3cf3f53151 100644
--- a/flang/test/Preprocessing/pp112.F90
+++ b/flang/test/Preprocessing/pp112.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = iflm (666)
 ! ditto, but without & ! comment
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp113.F90 b/flang/test/Preprocessing/pp113.F90
index 56e34389068f..9604316cc238 100644
--- a/flang/test/Preprocessing/pp113.F90
+++ b/flang/test/Preprocessing/pp113.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((666)+111)
 ! FLM call split across continuation between name and (, leading &
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp114.F90 b/flang/test/Preprocessing/pp114.F90
index 4c44759ba96e..c44ca7d3924d 100644
--- a/flang/test/Preprocessing/pp114.F90
+++ b/flang/test/Preprocessing/pp114.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((666)+111)
 ! ditto, with & ! comment, leading &
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp115.F90 b/flang/test/Preprocessing/pp115.F90
index 290f62e68948..29b580241f38 100644
--- a/flang/test/Preprocessing/pp115.F90
+++ b/flang/test/Preprocessing/pp115.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = iflm (666)
 ! ditto, with & ! comment, no leading &
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp116.F90 b/flang/test/Preprocessing/pp116.F90
index d41fab771f84..9e070bac6bca 100644
--- a/flang/test/Preprocessing/pp116.F90
+++ b/flang/test/Preprocessing/pp116.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = iflm (666)
 ! FLM call split between name and (, no leading &
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp117.F90 b/flang/test/Preprocessing/pp117.F90
index c8d1c2ff3130..bb0692cd0b7b 100644
--- a/flang/test/Preprocessing/pp117.F90
+++ b/flang/test/Preprocessing/pp117.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777 .eq. 777) then
 ! KWM rescan
       integer, parameter :: KWM = 666, KWM2 = 667
diff --git a/flang/test/Preprocessing/pp118.F90 b/flang/test/Preprocessing/pp118.F90
index e71cc410823b..fe24d83406b5 100644
--- a/flang/test/Preprocessing/pp118.F90
+++ b/flang/test/Preprocessing/pp118.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(kwm2 .eq. 777) then
 ! KWM rescan with #undef, proving rescan after expansion
       integer, parameter :: KWM2 = 777, KWM = 667
diff --git a/flang/test/Preprocessing/pp119.F90 b/flang/test/Preprocessing/pp119.F90
index ea9a15cb02de..514bff0c981f 100644
--- a/flang/test/Preprocessing/pp119.F90
+++ b/flang/test/Preprocessing/pp119.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((666)+111)
 ! FLM rescan
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp120.F90 b/flang/test/Preprocessing/pp120.F90
index 9c4b88c8d15c..510550b33496 100644
--- a/flang/test/Preprocessing/pp120.F90
+++ b/flang/test/Preprocessing/pp120.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = ((111)+666)
 ! FLM expansion of argument
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp121.F90 b/flang/test/Preprocessing/pp121.F90
index cdf25e15ee44..900e255d50c6 100644
--- a/flang/test/Preprocessing/pp121.F90
+++ b/flang/test/Preprocessing/pp121.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: ch = 'KWM'
 ! CHECK: if(ch .eq. 'KWM') then
 ! KWM NOT expanded in 'literal'
diff --git a/flang/test/Preprocessing/pp122.F90 b/flang/test/Preprocessing/pp122.F90
index 14c8762af1c7..8051cbd4c67c 100644
--- a/flang/test/Preprocessing/pp122.F90
+++ b/flang/test/Preprocessing/pp122.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: ch = "KWM"
 ! KWM NOT expanded in "literal"
 #define KWM 666
diff --git a/flang/test/Preprocessing/pp123.F90 b/flang/test/Preprocessing/pp123.F90
index 1768cec1c23b..8181b683a569 100644
--- a/flang/test/Preprocessing/pp123.F90
+++ b/flang/test/Preprocessing/pp123.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: ch = 3hKWM
 ! KWM NOT expanded in Hollerith literal
 #define KWM 666
diff --git a/flang/test/Preprocessing/pp124.F90 b/flang/test/Preprocessing/pp124.F90
index bb011515ce1c..750f9facd279 100644
--- a/flang/test/Preprocessing/pp124.F90
+++ b/flang/test/Preprocessing/pp124.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: 100 format(3hKWM)
 ! KWM NOT expanded in Hollerith in FORMAT
 #define KWM 666
diff --git a/flang/test/Preprocessing/pp125.F90 b/flang/test/Preprocessing/pp125.F90
index cf3909b88824..86abccca7b4e 100644
--- a/flang/test/Preprocessing/pp125.F90
+++ b/flang/test/Preprocessing/pp125.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777 .eq. 777) then
 ! #DEFINE works in free form
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp126.F90 b/flang/test/Preprocessing/pp126.F90
index fefb7fcf118f..f41af1e1b1cf 100644
--- a/flang/test/Preprocessing/pp126.F90
+++ b/flang/test/Preprocessing/pp126.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: if(777 .eq. 777) then
 ! \ newline works in #define
       integer, parameter :: KWM = 666
diff --git a/flang/test/Preprocessing/pp127.F90 b/flang/test/Preprocessing/pp127.F90
index 08feedf95305..09de8ede986b 100644
--- a/flang/test/Preprocessing/pp127.F90
+++ b/flang/test/Preprocessing/pp127.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = iflm(666 )
 ! FLM call with closing ')' on next line (not a continuation)
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp128.F90 b/flang/test/Preprocessing/pp128.F90
index 46918e05f5c3..a8bc79227aaa 100644
--- a/flang/test/Preprocessing/pp128.F90
+++ b/flang/test/Preprocessing/pp128.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK: res = iflm
 ! FLM call with '(' on next line (not a continuation)
       integer function IFLM(x)
diff --git a/flang/test/Preprocessing/pp129.F90 b/flang/test/Preprocessing/pp129.F90
index ab7e7f0baa8b..dee4c7b6d144 100644
--- a/flang/test/Preprocessing/pp129.F90
+++ b/flang/test/Preprocessing/pp129.F90
@@ -1,4 +1,4 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! RUN: %flang -E %s 2>&1 | FileCheck %s
 ! CHECK-NOT: stop
 ! #define KWM !, then KWM works as comment line initiator
 #define KWM !
diff --git a/flang/test/Preprocessing/pp130.F90 b/flang/test/Preprocessing/pp130.F90
index be1148807b8b..8066e28280e1 100644
--- a/flang/test/Preprocessing/pp130.F90
+++ b/flang/test/Preprocessing/pp130.F90
@@ -1,4 +1,4 @@
-! RUN: (%f18 -E %s 2>&1 || true) | FileCheck %s
+! RUN: (%flang -E %s 2>&1 || true) | FileCheck %s
 ! CHECK: error: bad character ('&') in Fortran token
 ! #define KWM &, use for continuation w/o pasting (ifort and nag seem to continue #define)
 #define KWM &
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index c9992b75c218..d724628dc0bb 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -73,6 +73,11 @@
 
 if config.include_flang_new_driver_test:
    tools.append(ToolSubst('%flang-new', command=FindTool('flang-new'), unresolved='fatal'))
+   tools.append(ToolSubst('%flang', command=FindTool('flang-new'), unresolved='fatal'))
+else:
+   tools.append(ToolSubst('%flang', command=FindTool('f18'),
+    extra_args=["-intrinsic-module-directory "+config.flang_intrinsic_modules_dir],
+    unresolved='fatal'))
 
 if config.flang_standalone_build:
     llvm_config.add_tool_substitutions(tools, [config.flang_llvm_tools_dir])
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 6a55245ab87a..4e7e8f978546 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -19,6 +19,8 @@ set(CMAKE_MODULE_PATH
   ${CMAKE_MODULE_PATH}
   )
 
+set(CMAKE_FOLDER "libc++")
+
 set(LIBCXX_SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR})
 set(LIBCXX_BINARY_DIR  ${CMAKE_CURRENT_BINARY_DIR})
 set(LIBCXX_BINARY_INCLUDE_DIR "${LIBCXX_BINARY_DIR}/include/c++build")
diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 42d25c20c811..0327f05ccfa6 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -5,6 +5,8 @@ include(CheckCXXCompilerFlag)
 # Build Google Benchmark for libc++
 #==============================================================================
 
+set(CMAKE_FOLDER "${CMAKE_FOLDER}/Benchmarks")
+
 set(BENCHMARK_LIBCXX_COMPILE_FLAGS
     -Wno-unused-command-line-argument
     -nostdinc++
diff --git a/libcxx/docs/Cxx2aStatusPaperStatus.csv b/libcxx/docs/Cxx2aStatusPaperStatus.csv
index efdcb1b67161..8991a03b2a5c 100644
--- a/libcxx/docs/Cxx2aStatusPaperStatus.csv
+++ b/libcxx/docs/Cxx2aStatusPaperStatus.csv
@@ -59,7 +59,7 @@
 "`P0595R2 <https://wg21.link/P0595R2>`__","CWG","P0595R2 std::is_constant_evaluated()","San Diego","|Complete|","9.0"
 "`P0602R4 <https://wg21.link/P0602R4>`__","LWG","variant and optional should propagate copy/move triviality","San Diego","|Complete|","8.0"
 "`P0608R3 <https://wg21.link/P0608R3>`__","LWG","A sane variant converting constructor","San Diego","|Complete|","9.0"
-"`P0655R1 <https://wg21.link/P0655R1>`__","LWG","visit<R>: Explicit Return Type for visit","San Diego","* *",""
+"`P0655R1 <https://wg21.link/P0655R1>`__","LWG","visit<R>: Explicit Return Type for visit","San Diego","|Complete|","12.0"
 "`P0771R1 <https://wg21.link/P0771R1>`__","LWG","std::function move constructor should be noexcept","San Diego","|Complete|","6.0"
 "`P0896R4 <https://wg21.link/P0896R4>`__","LWG","The One Ranges Proposal","San Diego","* *",""
 "`P0899R1 <https://wg21.link/P0899R1>`__","LWG","P0899R1 - LWG 3016 is not a defect","San Diego","|Nothing To Do|",""
diff --git a/libcxx/include/variant b/libcxx/include/variant
index daa3dd2d71c6..cb46f09bdc06 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -169,6 +169,9 @@ namespace std {
   template <class Visitor, class... Variants>
   constexpr see below visit(Visitor&&, Variants&&...);
 
+  template <class R, class Visitor, class... Variants>
+  constexpr R visit(Visitor&&, Variants&&...); // since C++20
+
   // 20.7.7, class monostate
   struct monostate;
 
@@ -583,6 +586,16 @@ struct __variant {
         __make_value_visitor(_VSTD::forward<_Visitor>(__visitor)),
         _VSTD::forward<_Vs>(__vs)...);
   }
+#if _LIBCPP_STD_VER > 17
+  template <class _Rp, class _Visitor, class... _Vs>
+  inline _LIBCPP_INLINE_VISIBILITY
+  static constexpr _Rp __visit_value(_Visitor&& __visitor,
+                                     _Vs&&... __vs) {
+    return __visit_alt(
+        __make_value_visitor<_Rp>(_VSTD::forward<_Visitor>(__visitor)),
+        _VSTD::forward<_Vs>(__vs)...);
+  }
+#endif
 
 private:
   template <class _Visitor, class... _Values>
@@ -605,12 +618,43 @@ private:
     _Visitor&& __visitor;
   };
 
+#if _LIBCPP_STD_VER > 17
+  template <class _Rp, class _Visitor>
+  struct __value_visitor_return_type {
+    template <class... _Alts>
+    inline _LIBCPP_INLINE_VISIBILITY
+    constexpr _Rp operator()(_Alts&&... __alts) const {
+      __std_visit_exhaustive_visitor_check<
+          _Visitor,
+          decltype((_VSTD::forward<_Alts>(__alts).__value))...>();
+      if constexpr (is_void_v<_Rp>) {
+        _VSTD::__invoke_constexpr(_VSTD::forward<_Visitor>(__visitor),
+                                  _VSTD::forward<_Alts>(__alts).__value...);
+      }
+      else {
+        return _VSTD::__invoke_constexpr(_VSTD::forward<_Visitor>(__visitor),
+                                         _VSTD::forward<_Alts>(__alts).__value...);
+      }
+    }
+
+    _Visitor&& __visitor;
+  };
+#endif
+
   template <class _Visitor>
   inline _LIBCPP_INLINE_VISIBILITY
   static constexpr auto __make_value_visitor(_Visitor&& __visitor) {
     return __value_visitor<_Visitor>{_VSTD::forward<_Visitor>(__visitor)};
   }
+
+#if _LIBCPP_STD_VER > 17
+  template <class _Rp, class _Visitor>
+  inline _LIBCPP_INLINE_VISIBILITY
+  static constexpr auto __make_value_visitor(_Visitor&& __visitor) {
+    return __value_visitor_return_type<_Rp, _Visitor>{_VSTD::forward<_Visitor>(__visitor)};
+  }
 };
+#endif
 
 } // namespace __visitation
 
@@ -1594,18 +1638,37 @@ constexpr bool operator>=(const variant<_Types...>& __lhs,
 template <class _Visitor, class... _Vs>
 inline _LIBCPP_INLINE_VISIBILITY
 _LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
-constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) {
-  using __variant_detail::__visitation::__variant;
-  bool __results[] = {__vs.valueless_by_exception()...};
-  for (bool __result : __results) {
-    if (__result) {
+constexpr void __throw_if_valueless(_Visitor&& __visitor, _Vs&&... __vs) {
+  const bool __valueless = (... || __vs.valueless_by_exception());
+  if (__valueless) {
       __throw_bad_variant_access();
-    }
   }
+}
+
+template <class _Visitor, class... _Vs>
+inline _LIBCPP_INLINE_VISIBILITY
+_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
+constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs) {
+  using __variant_detail::__visitation::__variant;
+  _VSTD::__throw_if_valueless(_VSTD::forward<_Visitor>(__visitor),
+                              _VSTD::forward<_Vs>(__vs)...);
   return __variant::__visit_value(_VSTD::forward<_Visitor>(__visitor),
                                   _VSTD::forward<_Vs>(__vs)...);
 }
 
+#if _LIBCPP_STD_VER > 17
+template <class _Rp, class _Visitor, class... _Vs>
+inline _LIBCPP_INLINE_VISIBILITY
+_LIBCPP_AVAILABILITY_THROW_BAD_VARIANT_ACCESS
+constexpr _Rp visit(_Visitor&& __visitor, _Vs&&... __vs) {
+  using __variant_detail::__visitation::__variant;
+  _VSTD::__throw_if_valueless(_VSTD::forward<_Visitor>(__visitor),
+                              _VSTD::forward<_Vs>(__vs)...);
+  return __variant::__visit_value<_Rp>(_VSTD::forward<_Visitor>(__visitor),
+                                       _VSTD::forward<_Vs>(__vs)...);
+}
+#endif
+
 struct _LIBCPP_TEMPLATE_VIS monostate {};
 
 inline _LIBCPP_INLINE_VISIBILITY
diff --git a/libcxx/test/std/utilities/variant/variant.visit/robust_against_adl.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/robust_against_adl.pass.cpp
index af3cd759dbef..94ca6cf81ff6 100644
--- a/libcxx/test/std/utilities/variant/variant.visit/robust_against_adl.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit/robust_against_adl.pass.cpp
@@ -32,6 +32,10 @@ constexpr bool test(bool do_it)
         std::variant<Holder<Incomplete>*, int> v = nullptr;
         std::visit([](auto){}, v);
         std::visit([](auto) -> Holder<Incomplete>* { return nullptr; }, v);
+#if TEST_STD_VER > 17
+        std::visit<void>([](auto){}, v);
+        std::visit<void*>([](auto) -> Holder<Incomplete>* { return nullptr; }, v);
+#endif
     }
     return true;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
index 0edb83be04d4..b88ecc037d51 100644
--- a/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
@@ -27,70 +27,8 @@
 #include <variant>
 
 #include "test_macros.h"
-#include "type_id.h"
 #include "variant_test_helpers.h"
 
-enum CallType : unsigned {
-  CT_None,
-  CT_NonConst = 1,
-  CT_Const = 2,
-  CT_LValue = 4,
-  CT_RValue = 8
-};
-
-inline constexpr CallType operator|(CallType LHS, CallType RHS) {
-  return static_cast<CallType>(static_cast<unsigned>(LHS) |
-                               static_cast<unsigned>(RHS));
-}
-
-struct ForwardingCallObject {
-
-  template <class... Args>
-  ForwardingCallObject& operator()(Args&&...) & {
-    set_call<Args &&...>(CT_NonConst | CT_LValue);
-    return *this;
-  }
-
-  template <class... Args>
-  const ForwardingCallObject& operator()(Args&&...) const & {
-    set_call<Args &&...>(CT_Const | CT_LValue);
-    return *this;
-  }
-
-  template <class... Args>
-  ForwardingCallObject&& operator()(Args&&...) && {
-    set_call<Args &&...>(CT_NonConst | CT_RValue);
-    return std::move(*this);
-  }
-
-  template <class... Args>
-  const ForwardingCallObject&& operator()(Args&&...) const && {
-    set_call<Args &&...>(CT_Const | CT_RValue);
-    return std::move(*this);
-  }
-
-  template <class... Args> static void set_call(CallType type) {
-    assert(last_call_type == CT_None);
-    assert(last_call_args == nullptr);
-    last_call_type = type;
-    last_call_args = std::addressof(makeArgumentID<Args...>());
-  }
-
-  template <class... Args> static bool check_call(CallType type) {
-    bool result = last_call_type == type && last_call_args &&
-                  *last_call_args == makeArgumentID<Args...>();
-    last_call_type = CT_None;
-    last_call_args = nullptr;
-    return result;
-  }
-
-  static CallType last_call_type;
-  static const TypeID *last_call_args;
-};
-
-CallType ForwardingCallObject::last_call_type = CT_None;
-const TypeID *ForwardingCallObject::last_call_args = nullptr;
-
 void test_call_operator_forwarding() {
   using Fn = ForwardingCallObject;
   Fn obj{};
@@ -296,18 +234,6 @@ void test_return_type() {
   }
 }
 
-struct ReturnFirst {
-  template <class... Args> constexpr int operator()(int f, Args &&...) const {
-    return f;
-  }
-};
-
-struct ReturnArity {
-  template <class... Args> constexpr int operator()(Args &&...) const {
-    return sizeof...(Args);
-  }
-};
-
 void test_constexpr() {
   constexpr ReturnFirst obj{};
   constexpr ReturnArity aobj{};
diff --git a/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
new file mode 100644
index 000000000000..7e569e2e9e42
--- /dev/null
+++ b/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
@@ -0,0 +1,430 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// Throwing bad_variant_access is supported starting in macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12 && !no-exceptions
+// XFAIL: with_system_cxx_lib=macosx10.11 && !no-exceptions
+// XFAIL: with_system_cxx_lib=macosx10.10 && !no-exceptions
+// XFAIL: with_system_cxx_lib=macosx10.9 && !no-exceptions
+
+// <variant>
+// template <class R, class Visitor, class... Variants>
+// constexpr R visit(Visitor&& vis, Variants&&... vars);
+
+#include <cassert>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+#include "test_macros.h"
+#include "variant_test_helpers.h"
+
+template <typename ReturnType>
+void test_call_operator_forwarding() {
+  using Fn = ForwardingCallObject;
+  Fn obj{};
+  const Fn &cobj = obj;
+  { // test call operator forwarding - no variant
+    std::visit<ReturnType>(obj);
+    assert(Fn::check_call<>(CT_NonConst | CT_LValue));
+    std::visit<ReturnType>(cobj);
+    assert(Fn::check_call<>(CT_Const | CT_LValue));
+    std::visit<ReturnType>(std::move(obj));
+    assert(Fn::check_call<>(CT_NonConst | CT_RValue));
+    std::visit<ReturnType>(std::move(cobj));
+    assert(Fn::check_call<>(CT_Const | CT_RValue));
+  }
+  { // test call operator forwarding - single variant, single arg
+    using V = std::variant<int>;
+    V v(42);
+    std::visit<ReturnType>(obj, v);
+    assert(Fn::check_call<int &>(CT_NonConst | CT_LValue));
+    std::visit<ReturnType>(cobj, v);
+    assert(Fn::check_call<int &>(CT_Const | CT_LValue));
+    std::visit<ReturnType>(std::move(obj), v);
+    assert(Fn::check_call<int &>(CT_NonConst | CT_RValue));
+    std::visit<ReturnType>(std::move(cobj), v);
+    assert(Fn::check_call<int &>(CT_Const | CT_RValue));
+  }
+  { // test call operator forwarding - single variant, multi arg
+    using V = std::variant<int, long, double>;
+    V v(42l);
+    std::visit<ReturnType>(obj, v);
+    assert(Fn::check_call<long &>(CT_NonConst | CT_LValue));
+    std::visit<ReturnType>(cobj, v);
+    assert(Fn::check_call<long &>(CT_Const | CT_LValue));
+    std::visit<ReturnType>(std::move(obj), v);
+    assert(Fn::check_call<long &>(CT_NonConst | CT_RValue));
+    std::visit<ReturnType>(std::move(cobj), v);
+    assert(Fn::check_call<long &>(CT_Const | CT_RValue));
+  }
+  { // test call operator forwarding - multi variant, multi arg
+    using V = std::variant<int, long, double>;
+    using V2 = std::variant<int *, std::string>;
+    V v(42l);
+    V2 v2("hello");
+    std::visit<int>(obj, v, v2);
+    assert((Fn::check_call<long &, std::string &>(CT_NonConst | CT_LValue)));
+    std::visit<ReturnType>(cobj, v, v2);
+    assert((Fn::check_call<long &, std::string &>(CT_Const | CT_LValue)));
+    std::visit<ReturnType>(std::move(obj), v, v2);
+    assert((Fn::check_call<long &, std::string &>(CT_NonConst | CT_RValue)));
+    std::visit<ReturnType>(std::move(cobj), v, v2);
+    assert((Fn::check_call<long &, std::string &>(CT_Const | CT_RValue)));
+  }
+  {
+    using V = std::variant<int, long, double, std::string>;
+    V v1(42l), v2("hello"), v3(101), v4(1.1);
+    std::visit<ReturnType>(obj, v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int &, double &>(CT_NonConst | CT_LValue)));
+    std::visit<ReturnType>(cobj, v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int &, double &>(CT_Const | CT_LValue)));
+    std::visit<ReturnType>(std::move(obj), v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int &, double &>(CT_NonConst | CT_RValue)));
+    std::visit<ReturnType>(std::move(cobj), v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int &, double &>(CT_Const | CT_RValue)));
+  }
+  {
+    using V = std::variant<int, long, double, int*, std::string>;
+    V v1(42l), v2("hello"), v3(nullptr), v4(1.1);
+    std::visit<ReturnType>(obj, v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int *&, double &>(CT_NonConst | CT_LValue)));
+    std::visit<ReturnType>(cobj, v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int *&, double &>(CT_Const | CT_LValue)));
+    std::visit<ReturnType>(std::move(obj), v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int *&, double &>(CT_NonConst | CT_RValue)));
+    std::visit<ReturnType>(std::move(cobj), v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int *&, double &>(CT_Const | CT_RValue)));
+  }
+}
+
+template <typename ReturnType>
+void test_argument_forwarding() {
+  using Fn = ForwardingCallObject;
+  Fn obj{};
+  const auto Val = CT_LValue | CT_NonConst;
+  { // single argument - value type
+    using V = std::variant<int>;
+    V v(42);
+    const V &cv = v;
+    std::visit<ReturnType>(obj, v);
+    assert(Fn::check_call<int &>(Val));
+    std::visit<ReturnType>(obj, cv);
+    assert(Fn::check_call<const int &>(Val));
+    std::visit<ReturnType>(obj, std::move(v));
+    assert(Fn::check_call<int &&>(Val));
+    std::visit<ReturnType>(obj, std::move(cv));
+    assert(Fn::check_call<const int &&>(Val));
+  }
+#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
+  { // single argument - lvalue reference
+    using V = std::variant<int &>;
+    int x = 42;
+    V v(x);
+    const V &cv = v;
+    std::visit<ReturnType>(obj, v);
+    assert(Fn::check_call<int &>(Val));
+    std::visit<ReturnType>(obj, cv);
+    assert(Fn::check_call<int &>(Val));
+    std::visit<ReturnType>(obj, std::move(v));
+    assert(Fn::check_call<int &>(Val));
+    std::visit<ReturnType>(obj, std::move(cv));
+    assert(Fn::check_call<int &>(Val));
+  }
+  { // single argument - rvalue reference
+    using V = std::variant<int &&>;
+    int x = 42;
+    V v(std::move(x));
+    const V &cv = v;
+    std::visit<ReturnType>(obj, v);
+    assert(Fn::check_call<int &>(Val));
+    std::visit<ReturnType>(obj, cv);
+    assert(Fn::check_call<int &>(Val));
+    std::visit<ReturnType>(obj, std::move(v));
+    assert(Fn::check_call<int &&>(Val));
+    std::visit<ReturnType>(obj, std::move(cv));
+    assert(Fn::check_call<int &&>(Val));
+  }
+#endif
+  { // multi argument - multi variant
+    using V = std::variant<int, std::string, long>;
+    V v1(42), v2("hello"), v3(43l);
+    std::visit<ReturnType>(obj, v1, v2, v3);
+    assert((Fn::check_call<int &, std::string &, long &>(Val)));
+    std::visit<ReturnType>(obj, std::as_const(v1), std::as_const(v2), std::move(v3));
+    assert((Fn::check_call<const int &, const std::string &, long &&>(Val)));
+  }
+  {
+    using V = std::variant<int, long, double, std::string>;
+    V v1(42l), v2("hello"), v3(101), v4(1.1);
+    std::visit<ReturnType>(obj, v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int &, double &>(Val)));
+    std::visit<ReturnType>(obj, std::as_const(v1), std::as_const(v2), std::move(v3), std::move(v4));
+    assert((Fn::check_call<const long &, const std::string &, int &&, double &&>(Val)));
+  }
+  {
+    using V = std::variant<int, long, double, int*, std::string>;
+    V v1(42l), v2("hello"), v3(nullptr), v4(1.1);
+    std::visit<ReturnType>(obj, v1, v2, v3, v4);
+    assert((Fn::check_call<long &, std::string &, int *&, double &>(Val)));
+    std::visit<ReturnType>(obj, std::as_const(v1), std::as_const(v2), std::move(v3), std::move(v4));
+    assert((Fn::check_call<const long &, const std::string &, int *&&, double &&>(Val)));
+  }
+}
+
+template <typename ReturnType>
+void test_return_type() {
+  using Fn = ForwardingCallObject;
+  Fn obj{};
+  const Fn &cobj = obj;
+  { // test call operator forwarding - no variant
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(obj)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(cobj)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(obj))), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(cobj))), ReturnType>);
+  }
+  { // test call operator forwarding - single variant, single arg
+    using V = std::variant<int>;
+    V v(42);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(obj, v)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(cobj, v)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(obj), v)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(cobj), v)), ReturnType>);
+  }
+  { // test call operator forwarding - single variant, multi arg
+    using V = std::variant<int, long, double>;
+    V v(42l);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(obj, v)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(cobj, v)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(obj), v)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(cobj), v)), ReturnType>);
+  }
+  { // test call operator forwarding - multi variant, multi arg
+    using V = std::variant<int, long, double>;
+    using V2 = std::variant<int *, std::string>;
+    V v(42l);
+    V2 v2("hello");
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(obj, v, v2)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(cobj, v, v2)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(obj), v, v2)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(cobj), v, v2)), ReturnType>);
+  }
+  {
+    using V = std::variant<int, long, double, std::string>;
+    V v1(42l), v2("hello"), v3(101), v4(1.1);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(obj, v1, v2, v3, v4)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(cobj, v1, v2, v3, v4)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(obj), v1, v2, v3, v4)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(cobj), v1, v2, v3, v4)), ReturnType>);
+  }
+  {
+    using V = std::variant<int, long, double, int*, std::string>;
+    V v1(42l), v2("hello"), v3(nullptr), v4(1.1);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(obj, v1, v2, v3, v4)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(cobj, v1, v2, v3, v4)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(obj), v1, v2, v3, v4)), ReturnType>);
+    static_assert(std::is_same_v<decltype(std::visit<ReturnType>(std::move(cobj), v1, v2, v3, v4)), ReturnType>);
+  }
+}
+
+void test_constexpr_void() {
+  constexpr ReturnFirst obj{};
+  constexpr ReturnArity aobj{};
+  {
+    using V = std::variant<int>;
+    constexpr V v(42);
+    static_assert((std::visit<void>(obj, v), 42) == 42, "");
+  }
+  {
+    using V = std::variant<short, long, char>;
+    constexpr V v(42l);
+    static_assert((std::visit<void>(obj, v), 42) == 42, "");
+  }
+  {
+    using V1 = std::variant<int>;
+    using V2 = std::variant<int, char *, long long>;
+    using V3 = std::variant<bool, int, int>;
+    constexpr V1 v1;
+    constexpr V2 v2(nullptr);
+    constexpr V3 v3;
+    static_assert((std::visit<void>(aobj, v1, v2, v3), 3) == 3, "");
+  }
+  {
+    using V1 = std::variant<int>;
+    using V2 = std::variant<int, char *, long long>;
+    using V3 = std::variant<void *, int, int>;
+    constexpr V1 v1;
+    constexpr V2 v2(nullptr);
+    constexpr V3 v3;
+    static_assert((std::visit<void>(aobj, v1, v2, v3), 3) == 3, "");
+  }
+  {
+    using V = std::variant<int, long, double, int *>;
+    constexpr V v1(42l), v2(101), v3(nullptr), v4(1.1);
+    static_assert((std::visit<void>(aobj, v1, v2, v3, v4), 4) == 4, "");
+  }
+  {
+    using V = std::variant<int, long, double, long long, int *>;
+    constexpr V v1(42l), v2(101), v3(nullptr), v4(1.1);
+    static_assert((std::visit<void>(aobj, v1, v2, v3, v4), 4) == 4, "");
+  }
+}
+
+void test_constexpr_int() {
+  constexpr ReturnFirst obj{};
+  constexpr ReturnArity aobj{};
+  {
+    using V = std::variant<int>;
+    constexpr V v(42);
+    static_assert(std::visit<int>(obj, v) == 42, "");
+  }
+  {
+    using V = std::variant<short, long, char>;
+    constexpr V v(42l);
+    static_assert(std::visit<int>(obj, v) == 42, "");
+  }
+  {
+    using V1 = std::variant<int>;
+    using V2 = std::variant<int, char *, long long>;
+    using V3 = std::variant<bool, int, int>;
+    constexpr V1 v1;
+    constexpr V2 v2(nullptr);
+    constexpr V3 v3;
+    static_assert(std::visit<int>(aobj, v1, v2, v3) == 3, "");
+  }
+  {
+    using V1 = std::variant<int>;
+    using V2 = std::variant<int, char *, long long>;
+    using V3 = std::variant<void *, int, int>;
+    constexpr V1 v1;
+    constexpr V2 v2(nullptr);
+    constexpr V3 v3;
+    static_assert(std::visit<int>(aobj, v1, v2, v3) == 3, "");
+  }
+  {
+    using V = std::variant<int, long, double, int *>;
+    constexpr V v1(42l), v2(101), v3(nullptr), v4(1.1);
+    static_assert(std::visit<int>(aobj, v1, v2, v3, v4) == 4, "");
+  }
+  {
+    using V = std::variant<int, long, double, long long, int *>;
+    constexpr V v1(42l), v2(101), v3(nullptr), v4(1.1);
+    static_assert(std::visit<int>(aobj, v1, v2, v3, v4) == 4, "");
+  }
+}
+
+template <typename ReturnType>
+void test_exceptions() {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  ReturnArity obj{};
+  auto test = [&](auto &&... args) {
+    try {
+      std::visit<ReturnType>(obj, args...);
+    } catch (const std::bad_variant_access &) {
+      return true;
+    } catch (...) {
+    }
+    return false;
+  };
+  {
+    using V = std::variant<int, MakeEmptyT>;
+    V v;
+    makeEmpty(v);
+    assert(test(v));
+  }
+  {
+    using V = std::variant<int, MakeEmptyT>;
+    using V2 = std::variant<long, std::string, void *>;
+    V v;
+    makeEmpty(v);
+    V2 v2("hello");
+    assert(test(v, v2));
+  }
+  {
+    using V = std::variant<int, MakeEmptyT>;
+    using V2 = std::variant<long, std::string, void *>;
+    V v;
+    makeEmpty(v);
+    V2 v2("hello");
+    assert(test(v2, v));
+  }
+  {
+    using V = std::variant<int, MakeEmptyT>;
+    using V2 = std::variant<long, std::string, void *, MakeEmptyT>;
+    V v;
+    makeEmpty(v);
+    V2 v2;
+    makeEmpty(v2);
+    assert(test(v, v2));
+  }
+  {
+    using V = std::variant<int, long, double, MakeEmptyT>;
+    V v1(42l), v2(101), v3(202), v4(1.1);
+    makeEmpty(v1);
+    assert(test(v1, v2, v3, v4));
+  }
+  {
+    using V = std::variant<int, long, double, long long, MakeEmptyT>;
+    V v1(42l), v2(101), v3(202), v4(1.1);
+    makeEmpty(v1);
+    makeEmpty(v2);
+    makeEmpty(v3);
+    makeEmpty(v4);
+    assert(test(v1, v2, v3, v4));
+  }
+#endif
+}
+
+// See https://bugs.llvm.org/show_bug.cgi?id=31916
+template <typename ReturnType>
+void test_caller_accepts_nonconst() {
+  struct A {};
+  struct Visitor {
+    auto operator()(A&) {
+      if constexpr (!std::is_void_v<ReturnType>)
+      {
+        return ReturnType{};
+      }
+    }
+  };
+  std::variant<A> v;
+  std::visit<ReturnType>(Visitor{}, v);
+}
+
+void test_constexpr_explicit_side_effect() {
+  auto test_lambda = [](int arg) constexpr {
+    std::variant<int> v = 101;
+    std::visit<void>([arg](int& x) constexpr { x = arg; }, v);
+    return std::get<int>(v);
+  };
+
+  static_assert(test_lambda(202) == 202, "");
+}
+
+int main(int, char**) {
+  test_call_operator_forwarding<void>();
+  test_argument_forwarding<void>();
+  test_return_type<void>();
+  test_constexpr_void();
+  test_exceptions<void>();
+  test_caller_accepts_nonconst<void>();
+  test_call_operator_forwarding<int>();
+  test_argument_forwarding<int>();
+  test_return_type<int>();
+  test_constexpr_int();
+  test_exceptions<int>();
+  test_caller_accepts_nonconst<int>();
+  test_constexpr_explicit_side_effect();
+
+  return 0;
+}
diff --git a/libcxx/test/support/variant_test_helpers.h b/libcxx/test/support/variant_test_helpers.h
index 0e040931e0cd..92faf7988bc6 100644
--- a/libcxx/test/support/variant_test_helpers.h
+++ b/libcxx/test/support/variant_test_helpers.h
@@ -14,6 +14,7 @@
 #include <cassert>
 
 #include "test_macros.h"
+#include "type_id.h"
 
 #if TEST_STD_VER <= 14
 #error This file requires C++17
@@ -85,5 +86,83 @@ void makeEmpty(Variant& v) {
 }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
+enum CallType : unsigned {
+  CT_None,
+  CT_NonConst = 1,
+  CT_Const = 2,
+  CT_LValue = 4,
+  CT_RValue = 8
+};
+
+inline constexpr CallType operator|(CallType LHS, CallType RHS) {
+  return static_cast<CallType>(static_cast<unsigned>(LHS) |
+                               static_cast<unsigned>(RHS));
+}
+
+struct ForwardingCallObject {
+
+  template <class... Args>
+  ForwardingCallObject& operator()(Args&&...) & {
+    set_call<Args &&...>(CT_NonConst | CT_LValue);
+    return *this;
+  }
+
+  template <class... Args>
+  const ForwardingCallObject& operator()(Args&&...) const & {
+    set_call<Args &&...>(CT_Const | CT_LValue);
+    return *this;
+  }
+
+  template <class... Args>
+  ForwardingCallObject&& operator()(Args&&...) && {
+    set_call<Args &&...>(CT_NonConst | CT_RValue);
+    return std::move(*this);
+  }
+
+  template <class... Args>
+  const ForwardingCallObject&& operator()(Args&&...) const && {
+    set_call<Args &&...>(CT_Const | CT_RValue);
+    return std::move(*this);
+  }
+
+  template <class... Args> static void set_call(CallType type) {
+    assert(last_call_type == CT_None);
+    assert(last_call_args == nullptr);
+    last_call_type = type;
+    last_call_args = std::addressof(makeArgumentID<Args...>());
+  }
+
+  template <class... Args> static bool check_call(CallType type) {
+    bool result = last_call_type == type && last_call_args &&
+                  *last_call_args == makeArgumentID<Args...>();
+    last_call_type = CT_None;
+    last_call_args = nullptr;
+    return result;
+  }
+
+  // To check explicit return type for visit<R>
+  constexpr operator int() const
+  {
+    return 0;
+  }
+
+  static CallType last_call_type;
+  static const TypeID *last_call_args;
+};
+
+CallType ForwardingCallObject::last_call_type = CT_None;
+const TypeID *ForwardingCallObject::last_call_args = nullptr;
+
+struct ReturnFirst {
+  template <class... Args> constexpr int operator()(int f, Args &&...) const {
+    return f;
+  }
+};
+
+struct ReturnArity {
+  template <class... Args> constexpr int operator()(Args &&...) const {
+    return sizeof...(Args);
+  }
+};
 
 #endif // SUPPORT_VARIANT_TEST_HELPERS_H
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index ede8bd71da4c..b803347c2a8e 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -17,6 +17,8 @@ set(CMAKE_MODULE_PATH
   ${CMAKE_MODULE_PATH}
   )
 
+set(CMAKE_FOLDER "libc++")
+
 set(LIBCXXABI_SOURCE_DIR  ${CMAKE_CURRENT_SOURCE_DIR})
 set(LIBCXXABI_BINARY_DIR  ${CMAKE_CURRENT_BINARY_DIR})
 set(LIBCXXABI_LIBCXX_PATH "${CMAKE_CURRENT_LIST_DIR}/../libcxx" CACHE PATH
diff --git a/lld/test/COFF/driver.test b/lld/test/COFF/driver.test
index 951a4d3319f1..d756dc9ed4aa 100644
--- a/lld/test/COFF/driver.test
+++ b/lld/test/COFF/driver.test
@@ -1,6 +1,6 @@
 # RUN: not lld-link nosuchfile.obj >& %t.log
 # RUN: FileCheck -check-prefix=MISSING %s < %t.log
-MISSING: 'nosuchfile.obj': {{.*}}{{[Nn]}}o such file or directory
+MISSING: 'nosuchfile.obj': {{[Nn]}}o such file or directory
 
 # RUN: lld-link --version | FileCheck -check-prefix=VERSION %s
 VERSION: {{LLD [0-9]+\.[0-9]+}}
diff --git a/lld/test/COFF/manifestinput-error.test b/lld/test/COFF/manifestinput-error.test
index 6b86b87ad3e8..ea61bae2f7c0 100644
--- a/lld/test/COFF/manifestinput-error.test
+++ b/lld/test/COFF/manifestinput-error.test
@@ -7,4 +7,4 @@
 # RUN:   /manifestuac:"level='requireAdministrator'" \
 # RUN:   /manifestinput:%p/Inputs/manifestinput.test %t.obj 2>&1 | FileCheck %s
 
-# CHECK: error: unable to find mt.exe in PATH: {{.*}}{{[Nn]}}o such file or directory
+# CHECK: error: unable to find mt.exe in PATH: {{[Nn]}}o such file or directory
diff --git a/lld/test/COFF/nodefaultlib.test b/lld/test/COFF/nodefaultlib.test
index 01e727dbffa3..61a31c21b6cc 100644
--- a/lld/test/COFF/nodefaultlib.test
+++ b/lld/test/COFF/nodefaultlib.test
@@ -19,8 +19,8 @@
 # RUN:   /nodefaultlib:std64.lib >& %t.log || true
 # RUN: FileCheck -check-prefix=CHECK3 %s < %t.log
 
-CHECK1: error: could not open 'hello64.obj': {{.*}}{{[Nn]}}o such file or directory
-CHECK2: error: could not open 'hello64': {{.*}}{{[Nn]}}o such file or directory
+CHECK1: error: could not open 'hello64.obj': {{[Nn]}}o such file or directory
+CHECK2: error: could not open 'hello64': {{[Nn]}}o such file or directory
 CHECK3: error: undefined symbol: MessageBoxA
 CHECK3-NEXT: >>> referenced by {{.*}}hello64.obj:(main)
 
diff --git a/lld/test/COFF/pdb-type-server-invalid-signature.yaml b/lld/test/COFF/pdb-type-server-invalid-signature.yaml
index 3c34f96d14a8..612bd74eeaa8 100644
--- a/lld/test/COFF/pdb-type-server-invalid-signature.yaml
+++ b/lld/test/COFF/pdb-type-server-invalid-signature.yaml
@@ -22,8 +22,8 @@
 # RUN: cp %S/Inputs/pdb-diff-cl.pdb %T
 # RUN: lld-link %t3.obj -out:%t3.exe -debug -pdb:%t3.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s -check-prefix=INVALID-PATH -allow-empty
 
-# INVALID-PATH-NOT: warning: Cannot use debug info for '{{.*}}3.obj' [LNK4099]
-# INVALID-PATH-NOT: failed to load reference 'c:\some_invalid_path_AABB98765\pdb-diff-cl.pdb': {{.*}}{{[Nn]}}o such file or directory
+# INVALID-PATH-NOT: warning: Cannot use debug info for '{{.*}}3.obj' [LNK4099]
+# INVALID-PATH-NOT: failed to load reference 'c:\some_invalid_path_AABB98765\pdb-diff-cl.pdb': {{[Nn]}}o such file or directory
 
 --- !COFF
 header:
diff --git a/lld/test/ELF/archive-thin-missing-member.s b/lld/test/ELF/archive-thin-missing-member.s
index 9a8e9bba7ee6..4db1376cfaf2 100644
--- a/lld/test/ELF/archive-thin-missing-member.s
+++ b/lld/test/ELF/archive-thin-missing-member.s
@@ -9,17 +9,17 @@
 
 # Test error when loading symbols from missing thin archive member.
 # RUN: not ld.lld --entry=_Z1fi %t-no-syms.a -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR1
-# ERR1: {{.*}}-no-syms.a: could not get the buffer for a child of the archive: '{{.*}}.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR1: {{.*}}-no-syms.a: could not get the buffer for a child of the archive: '{{.*}}.o': {{[Nn]}}o such file or directory
 
 # Test error when thin archive has symbol table but member is missing.
 # RUN: not ld.lld --entry=_Z1fi -m elf_amd64_fbsd %t-syms.a -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR2
-# ERR2: {{.*}}-syms.a: could not get the buffer for the member defining symbol f(int): '{{.*}}.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR2: {{.*}}-syms.a: could not get the buffer for the member defining symbol f(int): '{{.*}}.o': {{[Nn]}}o such file or directory
 # RUN: not ld.lld --entry=_Z1fi --no-demangle -m elf_amd64_fbsd %t-syms.a -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR2MANGLE
-# ERR2MANGLE: {{.*}}-syms.a: could not get the buffer for the member defining symbol _Z1fi: '{{.*}}.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR2MANGLE: {{.*}}-syms.a: could not get the buffer for the member defining symbol _Z1fi: '{{.*}}.o': {{[Nn]}}o such file or directory
 
 # Test error when thin archive is linked using --whole-archive but member is missing.
 # RUN: not ld.lld --entry=_Z1fi --whole-archive %t-syms.a -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR3
-# ERR3: {{.*}}-syms.a: could not get the buffer for a child of the archive: '{{.*}}.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR3: {{.*}}-syms.a: could not get the buffer for a child of the archive: '{{.*}}.o': {{[Nn]}}o such file or directory
 
 .global _Z1fi
 _Z1fi:
diff --git a/lld/test/ELF/basic.s b/lld/test/ELF/basic.s
index e952472d08f0..f6954b143493 100644
--- a/lld/test/ELF/basic.s
+++ b/lld/test/ELF/basic.s
@@ -220,7 +220,7 @@ _start:
 
 # RUN: not ld.lld %t.foo -o /dev/null 2>&1 | \
 # RUN:  FileCheck --check-prefix=MISSING %s
-# MISSING: cannot open {{.*}}.foo: {{.*}}{{[Nn]}}o such file or directory
+# MISSING: cannot open {{.*}}.foo: {{[Nn]}}o such file or directory
 
 # RUN: not ld.lld -o /dev/null 2>&1 | \
 # RUN:  FileCheck --check-prefix=NO_INPUT %s
@@ -228,7 +228,7 @@ _start:
 
 # RUN: not ld.lld %t.no.such.file -o /dev/null 2>&1 | \
 # RUN:  FileCheck --check-prefix=CANNOT_OPEN %s
-# CANNOT_OPEN: cannot open {{.*}}.no.such.file: {{.*}}{{[Nn]}}o such file or directory
+# CANNOT_OPEN: cannot open {{.*}}.no.such.file: {{[Nn]}}o such file or directory
 
 # RUN: not ld.lld --foo 2>&1 | FileCheck --check-prefix=UNKNOWN %s
 # UNKNOWN: unknown argument '--foo'
diff --git a/lld/test/ELF/symbol-ordering-file.s b/lld/test/ELF/symbol-ordering-file.s
index ae88c28f5fe4..f6776cdb6f67 100644
--- a/lld/test/ELF/symbol-ordering-file.s
+++ b/lld/test/ELF/symbol-ordering-file.s
@@ -43,7 +43,7 @@
 # RUN: not ld.lld --symbol-ordering-file=%t.nonexistent %t.o -o %t3.out 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=ERR -DFILE=%t.nonexistent
 
-# ERR: error: cannot open [[FILE]]: {{.*}}{{[Nn]}}o such file or directory
+# ERR: error: cannot open [[FILE]]: {{[Nn]}}o such file or directory
 
 ## Show that an empty ordering file can be handled (symbols remain in their
 ## normal order).
diff --git a/lld/test/ELF/tls-le-weak-undef.s b/lld/test/ELF/tls-le-weak-undef.s
deleted file mode 100644
index fefda9da9816..000000000000
--- a/lld/test/ELF/tls-le-weak-undef.s
+++ /dev/null
@@ -1,18 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
-# RUN: echo '.tbss; .globl tls; tls:' | llvm-mc -filetype=obj -triple=x86_64 - -o %tdef.o
-# RUN: ld.lld %t.o -o - | llvm-objdump -d - | FileCheck %s
-
-## A weak symbol does not fetch a lazy definition.
-# RUN: ld.lld %t.o --start-lib %tdef.o --end-lib -o - | llvm-objdump -d - | FileCheck %s
-
-## Undefined TLS symbols arbitrarily resolve to 0.
-# CHECK:  leaq 16(%rax), %rdx
-
-# RUN: ld.lld -shared %tdef.o -o %tdef.so
-# RUN: not ld.lld %t.o %tdef.so -o /dev/null 2>&1 | FileCheck --check-prefix=COPYRELOC %s
-
-# COPYRELOC: symbol 'tls' has no type
-
-.weak tls
-leaq tls@tpoff+16(%rax), %rdx
diff --git a/lld/test/ELF/tls-weak-undef.s b/lld/test/ELF/tls-weak-undef.s
new file mode 100644
index 000000000000..19d1337c9fc8
--- /dev/null
+++ b/lld/test/ELF/tls-weak-undef.s
@@ -0,0 +1,42 @@
+# REQUIRES: x86
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/exec.s -o %texec.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/shared.s -o %tshared.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/ledef.s -o %tdef.o
+# RUN: ld.lld %texec.o -o %t.exec
+# RUN: ld.lld %tshared.o -o %t.shared --shared
+# RUN: llvm-objdump -d %t.exec | FileCheck %s --check-prefix=EXEC
+# RUN: llvm-objdump -d %t.shared | FileCheck %s --check-prefix=SHARED
+
+## An undefined weak TLS symbol does not fetch a lazy definition.
+# RUN: ld.lld %texec.o --start-lib %tdef.o --end-lib -o %tlazy
+# RUN: llvm-objdump -d %tlazy | FileCheck %s --check-prefix=EXEC
+
+## Undefined TLS symbols arbitrarily resolve to 0.
+# EXEC:   leaq 16(%rax), %rdx
+## Initial-exec references to undefined weak symbols can be relaxed to LE
+## references.
+# EXEC:   leaq 32(%rax), %rax
+# SHARED: leaq 48(%rax), %rcx
+
+# RUN: ld.lld -shared %tdef.o -o %tdef.so
+# RUN: not ld.lld %texec.o %tdef.so -o /dev/null 2>&1 | FileCheck --check-prefix=ERROR %s
+
+# ERROR: symbol 'le' has no type
+
+#--- ledef.s
+.tbss
+.globl le
+le:
+
+#--- exec.s
+.weak le
+leaq le@tpoff+16(%rax), %rdx
+
+.weak ie
+addq ie@gottpoff+32(%rip), %rax
+
+#--- shared.s
+.weak ld
+leaq ld@dtpoff+48(%rax), %rcx
diff --git a/lldb/source/Plugins/Process/Utility/AuxVector.cpp b/lldb/source/Plugins/Process/Utility/AuxVector.cpp
index 685d9d0824f6..c4f45f759a33 100644
--- a/lldb/source/Plugins/Process/Utility/AuxVector.cpp
+++ b/lldb/source/Plugins/Process/Utility/AuxVector.cpp
@@ -82,6 +82,7 @@ const char *AuxVector::GetEntryName(EntryType type) const {
     case ENTRY_NAME(AUXV_AT_SECURE);         break;
     case ENTRY_NAME(AUXV_AT_BASE_PLATFORM);  break;
     case ENTRY_NAME(AUXV_AT_RANDOM);         break;
+    case ENTRY_NAME(AUXV_AT_HWCAP2);         break;
     case ENTRY_NAME(AUXV_AT_EXECFN);         break;
     case ENTRY_NAME(AUXV_AT_SYSINFO);        break;
     case ENTRY_NAME(AUXV_AT_SYSINFO_EHDR);   break;
diff --git a/lldb/source/Plugins/Process/Utility/AuxVector.h b/lldb/source/Plugins/Process/Utility/AuxVector.h
index c8c8b1249413..07a0010e198f 100644
--- a/lldb/source/Plugins/Process/Utility/AuxVector.h
+++ b/lldb/source/Plugins/Process/Utility/AuxVector.h
@@ -50,6 +50,7 @@ class AuxVector {
     AUXV_AT_SECURE = 23,        ///< Boolean, was exec setuid-like?
     AUXV_AT_BASE_PLATFORM = 24, ///< String identifying real platforms.
     AUXV_AT_RANDOM = 25,        ///< Address of 16 random bytes.
+    AUXV_AT_HWCAP2 = 26,        ///< Extension of AT_HWCAP.
     AUXV_AT_EXECFN = 31,        ///< Filename of executable.
     AUXV_AT_SYSINFO = 32, ///< Pointer to the global system page used for system
                           /// calls and other nice things.
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
index 1cbed5acb41c..2929f2009dd9 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
@@ -85,7 +85,7 @@ class RegisterInfoPOSIX_arm64
 
   size_t GetRegisterSetFromRegisterIndex(uint32_t reg_index) const override;
 
-  uint32_t ConfigureVectorRegisterInfos(uint32_t mode);
+  uint32_t ConfigureVectorRegisterInfos(uint32_t sve_vq);
 
   bool VectorSizeIsValid(uint32_t vq) {
     if (vq >= eVectorQuadwordAArch64 && vq <= eVectorQuadwordAArch64SVEMax)
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h b/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h
index ea43ef8fe457..9551db7e8ebf 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h
@@ -9,9 +9,6 @@
 #ifdef DECLARE_REGISTER_INFOS_ARM64_STRUCT
 
 enum {
-  sve_fpsr = fpu_fpsr,
-  sve_fpcr = fpu_fpcr,
-
   sve_vg = exc_far,
 
   sve_z0,
diff --git a/lldb/test/API/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py b/lldb/test/API/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py
index 8ddab260b494..74b5f2e16eeb 100644
--- a/lldb/test/API/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py
+++ b/lldb/test/API/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py
@@ -12,6 +12,7 @@ class TestPlatformProcessConnect(gdbremote_testcase.GdbRemoteTestCaseBase):
     @expectedFailureAll(hostoslist=["windows"], triple='.*-android')
     @skipIfWindows # lldb-server does not terminate correctly
     @skipIfDarwin # lldb-server not found correctly
+    @skipIf(oslist=["linux"], archs=["arm", "aarch64"])  # Fails randomly
     def test_platform_process_connect(self):
         self.build()
 
diff --git a/llvm/docs/tutorial/LangImpl01.rst b/llvm/docs/tutorial/LangImpl01.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl01.rst
+++ b/llvm/docs/tutorial/LangImpl01.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl02.rst b/llvm/docs/tutorial/LangImpl02.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl02.rst
+++ b/llvm/docs/tutorial/LangImpl02.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl03.rst b/llvm/docs/tutorial/LangImpl03.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl03.rst
+++ b/llvm/docs/tutorial/LangImpl03.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl04.rst b/llvm/docs/tutorial/LangImpl04.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl04.rst
+++ b/llvm/docs/tutorial/LangImpl04.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl05.rst b/llvm/docs/tutorial/LangImpl05.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl05.rst
+++ b/llvm/docs/tutorial/LangImpl05.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl06.rst b/llvm/docs/tutorial/LangImpl06.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl06.rst
+++ b/llvm/docs/tutorial/LangImpl06.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl07.rst b/llvm/docs/tutorial/LangImpl07.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl07.rst
+++ b/llvm/docs/tutorial/LangImpl07.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl08.rst b/llvm/docs/tutorial/LangImpl08.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl08.rst
+++ b/llvm/docs/tutorial/LangImpl08.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl09.rst b/llvm/docs/tutorial/LangImpl09.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl09.rst
+++ b/llvm/docs/tutorial/LangImpl09.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/LangImpl10.rst b/llvm/docs/tutorial/LangImpl10.rst
index defa9d7f216a..6b90634055a0 100644
--- a/llvm/docs/tutorial/LangImpl10.rst
+++ b/llvm/docs/tutorial/LangImpl10.rst
@@ -4,4 +4,4 @@
 Kaleidoscope Tutorial
 =====================
 
-The Kaleidoscope Tutorial has `moved to another location <MyFirstLanguageFrontend/index.html>`_ .
+The Kaleidoscope Tutorial has moved to :doc:`MyFirstLanguageFrontend/index`.
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst
index 78aa31b30cd5..98ff8711b1ac 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst
@@ -642,7 +642,7 @@ the LLVM JIT and optimizer. To build this example, use:
 .. code-block:: bash
 
     # Compile
-    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core mcjit native` -O3 -o toy
+    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core orcjit native` -O3 -o toy
     # Run
     ./toy
 
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl05.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl05.rst
index a55cfe277de5..cc7ffe58918a 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl05.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl05.rst
@@ -355,12 +355,12 @@ beginning of the block. :)
 Once the insertion point is set, we recursively codegen the "then"
 expression from the AST. To finish off the "then" block, we create an
 unconditional branch to the merge block. One interesting (and very
-important) aspect of the LLVM IR is that it `requires all basic blocks
-to be "terminated" <../LangRef.html#functionstructure>`_ with a `control
-flow instruction <../LangRef.html#terminators>`_ such as return or
-branch. This means that all control flow, *including fall throughs* must
-be made explicit in the LLVM IR. If you violate this rule, the verifier
-will emit an error.
+important) aspect of the LLVM IR is that it :ref:`requires all basic
+blocks to be "terminated" <functionstructure>` with a :ref:`control
+flow instruction <terminators>`  such as return or branch. This means
+that all control flow, *including fall throughs* must be made explicit
+in the LLVM IR. If you violate this rule, the verifier will emit an
+error.
 
 The final line here is quite subtle, but is very important. The basic
 issue is that when we create the Phi node in the merge block, we need to
@@ -801,7 +801,7 @@ the if/then/else and for expressions. To build this example, use:
 .. code-block:: bash
 
     # Compile
-    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core mcjit native` -O3 -o toy
+    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core orcjit native` -O3 -o toy
     # Run
     ./toy
 
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl06.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl06.rst
index 87c7a3753b37..4e6d0019056c 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl06.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl06.rst
@@ -747,7 +747,7 @@ the support for user-defined operators. To build this example, use:
 .. code-block:: bash
 
     # Compile
-    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core mcjit native` -O3 -o toy
+    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core orcjit native` -O3 -o toy
     # Run
     ./toy
 
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl07.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl07.rst
index d7abeeac8053..9ae39729e06c 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl07.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl07.rst
@@ -870,7 +870,7 @@ mutable variables and var/in support. To build this example, use:
 .. code-block:: bash
 
     # Compile
-    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core mcjit native` -O3 -o toy
+    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core orcjit native` -O3 -o toy
     # Run
     ./toy
 
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst
index c641999b226b..2ddd860b69d1 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl09.rst
@@ -452,7 +452,7 @@ debug information. To build this example, use:
 .. code-block:: bash
 
     # Compile
-    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core mcjit native` -O3 -o toy
+    clang++ -g toy.cpp `llvm-config --cxxflags --ldflags --system-libs --libs core orcjit native` -O3 -o toy
     # Run
     ./toy
 
diff --git a/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp b/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
index c71c4d9293ab..170a89913605 100644
--- a/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
+++ b/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
@@ -1,10 +1,28 @@
-//===-- examples/HowToUseJIT/HowToUseJIT.cpp - An example use of the JIT --===//
+//===------- HowToUseLLJIT.cpp - An example use of ORC-based LLJIT --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+//  This small program provides an example of how to quickly build a small
+//  module with a 'add1' function and use of IRBuilder to create add & return
+//  instructions.
+//
+// Goal:
+//  The goal of this snippet is to create in the memory
+//  the LLVM module consisting of a function as follow:
+//
+// int add1(int x) {
+//   return x+1;
+// }
+//  add1(42);
+//
+// then compile the module via LLJIT, then execute the 'add1'
+// function and return result to a driver, i.e. to a "host program".
+//
+//===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/IR/Function.h"
diff --git a/llvm/examples/Kaleidoscope/MCJIT/README.txt b/llvm/examples/Kaleidoscope/MCJIT/README.txt
index ba742645acc1..54c14ced1c83 100644
--- a/llvm/examples/Kaleidoscope/MCJIT/README.txt
+++ b/llvm/examples/Kaleidoscope/MCJIT/README.txt
@@ -2,11 +2,8 @@
 //                          Kaleidoscope with MCJIT
 //===----------------------------------------------------------------------===//
 
-The files in this directory are meant to accompany a series of blog posts
-that describe the process of porting the Kaleidoscope tutorial to use the MCJIT
-execution engine instead of the older JIT engine.
+The files in this directory are accompany a series of blog posts that describe 
+the process of porting the Kaleidoscope tutorial to use the MCJIT execution 
+engine instead of the older JIT engine.
 
-When the blog posts are ready this file will be updated with links to the posts.
 
-These directories contain Makefiles that allow the code to be built in a
-standalone manner, independent of the larger LLVM build infrastructure.
\ No newline at end of file
diff --git a/llvm/examples/Kaleidoscope/MCJIT/cached/README.txt b/llvm/examples/Kaleidoscope/MCJIT/cached/README.txt
index 6acaf057f286..55a283d52ac4 100644
--- a/llvm/examples/Kaleidoscope/MCJIT/cached/README.txt
+++ b/llvm/examples/Kaleidoscope/MCJIT/cached/README.txt
@@ -2,11 +2,12 @@
 //                          Kaleidoscope with MCJIT
 //===----------------------------------------------------------------------===//
 
-The files in this directory are meant to accompany the first in a series of
+The files in this directory are meant to accompany the third blog in a series of
 three blog posts that describe the process of porting the Kaleidoscope tutorial
 to use the MCJIT execution engine instead of the older JIT engine.
 
-When the blog post is ready this file will be updated with a link to the post.
+The link of blog post-
+https://blog.llvm.org/posts/2013-08-02-object-caching-with-kaleidoscope/
 
 The source code in this directory demonstrates the third version of the
 program, now modified to accept an input IR file on the command line and,
@@ -15,14 +16,12 @@ optionally, to use a basic caching mechanism to store generated object images.
 The toy-jit.cpp file contains a version of the original JIT-based source code
 that has been modified to support the input IR file command line option.
 
-This directory contain a Makefile that allow the code to be built in a
-standalone manner, independent of the larger LLVM build infrastructure. To build
-the program you will need to have 'clang++' and 'llvm-config' in your path. If
-you attempt to build using the LLVM 3.3 release, some minor modifications will
-be required.
+To build the program you will need to have 'clang++' and 'llvm-config' in your 
+path. If you attempt to build using the LLVM 3.3 release, some minor 
+modifications will be required.
 
 This directory also contains a Python script that may be used to generate random
 input for the program and test scripts to capture data for rough performance
 comparisons.  Another Python script will split generated input files into
 definitions and function calls for the purpose of testing the IR input and
-caching facilities.
\ No newline at end of file
+caching facilities.
diff --git a/llvm/examples/Kaleidoscope/MCJIT/complete/README.txt b/llvm/examples/Kaleidoscope/MCJIT/complete/README.txt
index 82bc397a4eb9..6bdd2b79033a 100644
--- a/llvm/examples/Kaleidoscope/MCJIT/complete/README.txt
+++ b/llvm/examples/Kaleidoscope/MCJIT/complete/README.txt
@@ -6,20 +6,16 @@ The files in this directory are meant to accompany the first in a series of
 three blog posts that describe the process of porting the Kaleidoscope tutorial
 to use the MCJIT execution engine instead of the older JIT engine.
 
-When the blog post is ready this file will be updated with a link to the post.
-
 The source code in this directory combines all previous versions, including the
 old JIT-based implementation, into a single file for easy comparison with
 command line options to select between the various possibilities.
 
-This directory contain a Makefile that allow the code to be built in a
-standalone manner, independent of the larger LLVM build infrastructure. To build
-the program you will need to have 'clang++' and 'llvm-config' in your path. If
-you attempt to build using the LLVM 3.3 release, some minor modifications will
-be required.
+To build the program you will need to have 'clang++' and 'llvm-config' in your 
+path. If you attempt to build using the LLVM 3.3 release, some minor 
+modifications will be required.
 
 This directory also contains a Python script that may be used to generate random
 input for the program and test scripts to capture data for rough performance
 comparisons.  Another Python script will split generated input files into
 definitions and function calls for the purpose of testing the IR input and
-caching facilities.
\ No newline at end of file
+caching facilities.
diff --git a/llvm/examples/Kaleidoscope/MCJIT/initial/README.txt b/llvm/examples/Kaleidoscope/MCJIT/initial/README.txt
index b352a784522f..4e973d4c3fd8 100644
--- a/llvm/examples/Kaleidoscope/MCJIT/initial/README.txt
+++ b/llvm/examples/Kaleidoscope/MCJIT/initial/README.txt
@@ -2,17 +2,16 @@
 //                          Kaleidoscope with MCJIT
 //===----------------------------------------------------------------------===//
 
-The files in this directory are meant to accompany the first in a series of
+The files in this directory are meant to accompany the first blog in a series of
 three blog posts that describe the process of porting the Kaleidoscope tutorial
 to use the MCJIT execution engine instead of the older JIT engine.
 
-When the blog post is ready this file will be updated with a link to the post.
+The link of blog post-
+https://blog.llvm.org/posts/2013-07-22-using-mcjit-with-kaleidoscope-tutorial/
 
 The source code in this directory demonstrates the initial working version of
 the program before subsequent performance improvements are applied.
 
-This directory contain a Makefile that allow the code to be built in a
-standalone manner, independent of the larger LLVM build infrastructure. To build
-the program you will need to have 'clang++' and 'llvm-config' in your path. If
-you attempt to build using the LLVM 3.3 release, some minor modifications will
-be required, as mentioned in the blog posts.
\ No newline at end of file
+To build the program you will need to have 'clang++' and 'llvm-config' in your 
+path. If you attempt to build using the LLVM 3.3 release, some minor 
+modifications will be required, as mentioned in the blog posts.
diff --git a/llvm/examples/Kaleidoscope/MCJIT/lazy/README.txt b/llvm/examples/Kaleidoscope/MCJIT/lazy/README.txt
index c5c271e02f69..403bd6af1282 100644
--- a/llvm/examples/Kaleidoscope/MCJIT/lazy/README.txt
+++ b/llvm/examples/Kaleidoscope/MCJIT/lazy/README.txt
@@ -2,11 +2,12 @@
 //                          Kaleidoscope with MCJIT
 //===----------------------------------------------------------------------===//
 
-The files in this directory are meant to accompany the first in a series of
+The files in this directory are meant to accompany the second blog in a series of
 three blog posts that describe the process of porting the Kaleidoscope tutorial
 to use the MCJIT execution engine instead of the older JIT engine.
 
-When the blog post is ready this file will be updated with a link to the post.
+The link of blog post-
+https://blog.llvm.org/posts/2013-07-29-kaleidoscope-performance-with-mcjit/
 
 The source code in this directory demonstrates the second version of the
 program, now modified to implement a sort of 'lazy' compilation.
@@ -14,12 +15,10 @@ program, now modified to implement a sort of 'lazy' compilation.
 The toy-jit.cpp file contains a version of the original JIT-based source code
 that has been modified to disable most stderr output for timing purposes.
 
-This directory contain a Makefile that allow the code to be built in a
-standalone manner, independent of the larger LLVM build infrastructure. To build
-the program you will need to have 'clang++' and 'llvm-config' in your path. If
-you attempt to build using the LLVM 3.3 release, some minor modifications will
-be required.
+To build the program you will need to have 'clang++' and 'llvm-config' in your 
+path. If you attempt to build using the LLVM 3.3 release, some minor 
+modifications will be required.
 
 This directory also contains a Python script that may be used to generate random
 input for the program and test scripts to capture data for rough performance
-comparisons.
\ No newline at end of file
+comparisons.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 1bc5377e6863..c3221aac8eea 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3117,16 +3117,6 @@ class TargetLoweringBase {
   MachineBasicBlock *emitPatchPoint(MachineInstr &MI,
                                     MachineBasicBlock *MBB) const;
 
-  /// Replace/modify the XRay custom event operands with target-dependent
-  /// details.
-  MachineBasicBlock *emitXRayCustomEvent(MachineInstr &MI,
-                                         MachineBasicBlock *MBB) const;
-
-  /// Replace/modify the XRay typed event operands with target-dependent
-  /// details.
-  MachineBasicBlock *emitXRayTypedEvent(MachineInstr &MI,
-                                        MachineBasicBlock *MBB) const;
-
   bool IsStrictFPEnabled;
 };
 
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index 98ba8cda372b..cae3b435f3b0 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -128,12 +128,6 @@ struct SectionHeader {
   StringRef Name;
 };
 
-struct SectionHeaderTable {
-  Optional<std::vector<SectionHeader>> Sections;
-  Optional<std::vector<SectionHeader>> Excluded;
-  Optional<bool> NoHeaders;
-};
-
 struct Symbol {
   StringRef Name;
   ELF_STT Type;
@@ -196,18 +190,26 @@ struct Chunk {
     ARMIndexTable,
     MipsABIFlags,
     Addrsig,
-    Fill,
     LinkerOptions,
     DependentLibraries,
     CallGraphProfile,
-    BBAddrMap
+    BBAddrMap,
+
+    // Special chunks.
+    SpecialChunksStart,
+    Fill = SpecialChunksStart,
+    SectionHeaderTable,
   };
 
   ChunkKind Kind;
   StringRef Name;
   Optional<llvm::yaml::Hex64> Offset;
 
-  Chunk(ChunkKind K) : Kind(K) {}
+  // Usually chunks are not created implicitly, but rather loaded from YAML.
+  // This flag is used to signal whether this is the case or not.
+  bool IsImplicit;
+
+  Chunk(ChunkKind K, bool Implicit) : Kind(K), IsImplicit(Implicit) {}
   virtual ~Chunk();
 };
 
@@ -222,17 +224,14 @@ struct Section : public Chunk {
   Optional<yaml::BinaryRef> Content;
   Optional<llvm::yaml::Hex64> Size;
 
-  // Usually sections are not created implicitly, but loaded from YAML.
-  // When they are, this flag is used to signal about that.
-  bool IsImplicit;
-
   // Holds the original section index.
   unsigned OriginalSecNdx;
 
-  Section(ChunkKind Kind, bool IsImplicit = false)
-      : Chunk(Kind), IsImplicit(IsImplicit) {}
+  Section(ChunkKind Kind, bool IsImplicit = false) : Chunk(Kind, IsImplicit) {}
 
-  static bool classof(const Chunk *S) { return S->Kind != ChunkKind::Fill; }
+  static bool classof(const Chunk *S) {
+    return S->Kind < ChunkKind::SpecialChunksStart;
+  }
 
   // Some derived sections might have their own special entries. This method
   // returns a vector of <entry name, is used> pairs. It is used for section
@@ -276,11 +275,34 @@ struct Fill : Chunk {
   Optional<yaml::BinaryRef> Pattern;
   llvm::yaml::Hex64 Size;
 
-  Fill() : Chunk(ChunkKind::Fill) {}
+  Fill() : Chunk(ChunkKind::Fill, /*Implicit=*/false) {}
 
   static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Fill; }
 };
 
+struct SectionHeaderTable : Chunk {
+  SectionHeaderTable(bool IsImplicit)
+      : Chunk(ChunkKind::SectionHeaderTable, IsImplicit) {}
+
+  static bool classof(const Chunk *S) {
+    return S->Kind == ChunkKind::SectionHeaderTable;
+  }
+
+  Optional<std::vector<SectionHeader>> Sections;
+  Optional<std::vector<SectionHeader>> Excluded;
+  Optional<bool> NoHeaders;
+
+  size_t getNumHeaders(size_t SectionsNum) const {
+    if (IsImplicit)
+      return SectionsNum;
+    if (NoHeaders)
+      return (*NoHeaders) ? 0 : SectionsNum;
+    return (Sections ? Sections->size() : 0) + /*Null section*/ 1;
+  }
+
+  static constexpr StringRef TypeStr = "SectionHeaderTable";
+};
+
 struct BBAddrMapSection : Section {
   Optional<std::vector<BBAddrMapEntry>> Entries;
 
@@ -665,7 +687,6 @@ struct ProgramHeader {
 
 struct Object {
   FileHeader Header;
-  Optional<SectionHeaderTable> SectionHeaders;
   std::vector<ProgramHeader> ProgramHeaders;
 
   // An object might contain output section descriptions as well as
@@ -688,6 +709,13 @@ struct Object {
     return Ret;
   }
 
+  const SectionHeaderTable &getSectionHeaderTable() const {
+    for (const std::unique_ptr<Chunk> &C : Chunks)
+      if (auto *S = dyn_cast<ELFYAML::SectionHeaderTable>(C.get()))
+        return *S;
+    llvm_unreachable("the section header table chunk must always be present");
+  }
+
   unsigned getMachine() const;
 };
 
@@ -837,11 +865,6 @@ struct MappingTraits<ELFYAML::FileHeader> {
   static void mapping(IO &IO, ELFYAML::FileHeader &FileHdr);
 };
 
-template <> struct MappingTraits<ELFYAML::SectionHeaderTable> {
-  static void mapping(IO &IO, ELFYAML::SectionHeaderTable &SecHdrTable);
-  static std::string validate(IO &IO, ELFYAML::SectionHeaderTable &SecHdrTable);
-};
-
 template <> struct MappingTraits<ELFYAML::SectionHeader> {
   static void mapping(IO &IO, ELFYAML::SectionHeader &SHdr);
 };
diff --git a/llvm/include/llvm/Support/InstructionCost.h b/llvm/include/llvm/Support/InstructionCost.h
index 725f8495ac09..fbc898b878bb 100644
--- a/llvm/include/llvm/Support/InstructionCost.h
+++ b/llvm/include/llvm/Support/InstructionCost.h
@@ -47,6 +47,7 @@ class InstructionCost {
 public:
   InstructionCost() = default;
 
+  InstructionCost(CostState) = delete;
   InstructionCost(CostType Val) : Value(Val), State(Valid) {}
 
   static InstructionCost getInvalid(CostType Val = 0) {
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index c1c7849fbe30..59d98054e3a2 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -708,8 +708,6 @@ bool MachineInstr::isCandidateForCallSiteEntry(QueryType Type) const {
   if (!isCall(Type))
     return false;
   switch (getOpcode()) {
-  case TargetOpcode::PATCHABLE_EVENT_CALL:
-  case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
   case TargetOpcode::PATCHPOINT:
   case TargetOpcode::STACKMAP:
   case TargetOpcode::STATEPOINT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index cb273a6f299c..615bea2a4905 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -959,13 +959,13 @@ static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy) {
   return false;
 }
 
-// Determines if it is a constant integer or a build vector of constant
+// Determines if it is a constant integer or a splat/build vector of constant
 // integers (and undefs).
 // Do not permit build vector implicit truncation.
 static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
   if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
     return !(Const->isOpaque() && NoOpaques);
-  if (N.getOpcode() != ISD::BUILD_VECTOR)
+  if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR)
     return false;
   unsigned BitWidth = N.getScalarValueSizeInBits();
   for (const SDValue &Op : N->op_values()) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 82da553954d2..2090762e2ff4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -306,7 +306,8 @@ bool ISD::matchUnaryPredicate(SDValue Op,
     return Match(Cst);
 
   // FIXME: Add support for vector UNDEF cases?
-  if (ISD::BUILD_VECTOR != Op.getOpcode())
+  if (ISD::BUILD_VECTOR != Op.getOpcode() &&
+      ISD::SPLAT_VECTOR != Op.getOpcode())
     return false;
 
   EVT SVT = Op.getValueType().getScalarType();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b9439a48d4b2..6638ff6a6358 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6785,7 +6785,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     // specific calling convention, and only for x86_64.
     // FIXME: Support other platforms later.
     const auto &Triple = DAG.getTarget().getTargetTriple();
-    if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
+    if (Triple.getArch() != Triple::x86_64)
       return;
 
     SDLoc DL = getCurSDLoc();
@@ -6816,7 +6816,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     // specific calling convention, and only for x86_64.
     // FIXME: Support other platforms later.
     const auto &Triple = DAG.getTarget().getTargetTriple();
-    if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
+    if (Triple.getArch() != Triple::x86_64)
       return;
 
     SDLoc DL = getCurSDLoc();
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7858bc6c43e4..71e7a49ff644 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5031,9 +5031,15 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
     return SDValue();
 
   SDValue Shift, Factor;
-  if (VT.isVector()) {
+  if (VT.isFixedLengthVector()) {
     Shift = DAG.getBuildVector(ShVT, dl, Shifts);
     Factor = DAG.getBuildVector(VT, dl, Factors);
+  } else if (VT.isScalableVector()) {
+    assert(Shifts.size() == 1 && Factors.size() == 1 &&
+           "Expected matchUnaryPredicate to return one element for scalable "
+           "vectors");
+    Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
+    Factor = DAG.getSplatVector(VT, dl, Factors[0]);
   } else {
     Shift = Shifts[0];
     Factor = Factors[0];
@@ -5126,11 +5132,20 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   SDValue MagicFactor, Factor, Shift, ShiftMask;
-  if (VT.isVector()) {
+  if (VT.isFixedLengthVector()) {
     MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
     Factor = DAG.getBuildVector(VT, dl, Factors);
     Shift = DAG.getBuildVector(ShVT, dl, Shifts);
     ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
+  } else if (VT.isScalableVector()) {
+    assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
+           Shifts.size() == 1 && ShiftMasks.size() == 1 &&
+           "Expected matchUnaryPredicate to return one element for scalable "
+           "vectors");
+    MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
+    Factor = DAG.getSplatVector(VT, dl, Factors[0]);
+    Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
+    ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
   } else {
     MagicFactor = MagicFactors[0];
     Factor = Factors[0];
@@ -5244,11 +5259,19 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   SDValue PreShift, PostShift, MagicFactor, NPQFactor;
-  if (VT.isVector()) {
+  if (VT.isFixedLengthVector()) {
     PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
     MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
     NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
     PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
+  } else if (VT.isScalableVector()) {
+    assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
+           NPQFactors.size() == 1 && PostShifts.size() == 1 &&
+           "Expected matchUnaryPredicate to return one for scalable vectors");
+    PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
+    MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
+    NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
+    PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
   } else {
     PreShift = PreShifts[0];
     MagicFactor = MagicFactors[0];
@@ -5300,8 +5323,10 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
   Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
   Created.push_back(Q.getNode());
 
+  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+
   SDValue One = DAG.getConstant(1, dl, VT);
-  SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
+  SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
   return DAG.getSelect(dl, VT, IsOne, N0, Q);
 }
 
@@ -5728,7 +5753,7 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
     return SDValue();
 
   SDValue PVal, AVal, KVal, QVal;
-  if (VT.isVector()) {
+  if (VT.isFixedLengthVector()) {
     if (HadOneDivisor) {
       // Try to turn PAmts into a splat, since we don't care about the values
       // that are currently '0'. If we can't, just keep '0'`s.
@@ -5747,6 +5772,15 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
     AVal = DAG.getBuildVector(VT, DL, AAmts);
     KVal = DAG.getBuildVector(ShVT, DL, KAmts);
     QVal = DAG.getBuildVector(VT, DL, QAmts);
+  } else if (VT.isScalableVector()) {
+    assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
+           QAmts.size() == 1 &&
+           "Expected matchUnaryPredicate to return one element for scalable "
+           "vectors");
+    PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
+    AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
+    KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
+    QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
   } else {
     PVal = PAmts[0];
     AVal = AAmts[0];
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 7aa569147da4..28c8bd0a7ded 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1227,36 +1227,6 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
   return MBB;
 }
 
-MachineBasicBlock *
-TargetLoweringBase::emitXRayCustomEvent(MachineInstr &MI,
-                                        MachineBasicBlock *MBB) const {
-  assert(MI.getOpcode() == TargetOpcode::PATCHABLE_EVENT_CALL &&
-         "Called emitXRayCustomEvent on the wrong MI!");
-  auto &MF = *MI.getMF();
-  auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc());
-  for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); ++OpIdx)
-    MIB.add(MI.getOperand(OpIdx));
-
-  MBB->insert(MachineBasicBlock::iterator(MI), MIB);
-  MI.eraseFromParent();
-  return MBB;
-}
-
-MachineBasicBlock *
-TargetLoweringBase::emitXRayTypedEvent(MachineInstr &MI,
-                                       MachineBasicBlock *MBB) const {
-  assert(MI.getOpcode() == TargetOpcode::PATCHABLE_TYPED_EVENT_CALL &&
-         "Called emitXRayTypedEvent on the wrong MI!");
-  auto &MF = *MI.getMF();
-  auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc());
-  for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); ++OpIdx)
-    MIB.add(MI.getOperand(OpIdx));
-
-  MBB->insert(MachineBasicBlock::iterator(MI), MIB);
-  MI.eraseFromParent();
-  return MBB;
-}
-
 /// findRepresentativeClass - Return the largest legal super-reg register class
 /// of the register class for the specified type and its associated "cost".
 // This function is in TargetLowering because it uses RegClassForVT which would
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index 3602601287f4..8b730bc23ce0 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -119,10 +119,9 @@ Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B,
 }
 
 EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName,
-                                   unsigned PointerSize, Edge::Kind Delta64,
-                                   Edge::Kind Delta32, Edge::Kind NegDelta32)
-    : EHFrameSectionName(EHFrameSectionName), PointerSize(PointerSize),
-      Delta64(Delta64), Delta32(Delta32), NegDelta32(NegDelta32) {}
+                                   Edge::Kind Delta64, Edge::Kind NegDelta32)
+    : EHFrameSectionName(EHFrameSectionName), Delta64(Delta64),
+      NegDelta32(NegDelta32) {}
 
 Error EHFrameEdgeFixer::operator()(LinkGraph &G) {
   auto *EHFrame = G.findSectionByName(EHFrameSectionName);
@@ -135,11 +134,6 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) {
     return Error::success();
   }
 
-  // Check that we support the graph's pointer size.
-  if (G.getPointerSize() != 4 && G.getPointerSize() != 8)
-    return make_error<JITLinkError>(
-        "EHFrameEdgeFixer only supports 32 and 64 bit targets");
-
   LLVM_DEBUG({
     dbgs() << "EHFrameEdgeFixer: Processing " << EHFrameSectionName << "...\n";
   });
@@ -264,6 +258,7 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) {
 Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B,
                                    size_t RecordOffset, size_t RecordLength,
                                    size_t CIEDeltaFieldOffset) {
+  using namespace dwarf;
 
   LLVM_DEBUG(dbgs() << "      Record is CIE\n");
 
@@ -334,12 +329,11 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B,
       uint8_t LSDAPointerEncoding;
       if (auto Err = RecordReader.readInteger(LSDAPointerEncoding))
         return Err;
-      if (!isSupportedPointerEncoding(LSDAPointerEncoding))
+      if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr))
         return make_error<JITLinkError>(
             "Unsupported LSDA pointer encoding " +
             formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " +
             formatv("{0:x16}", CIESymbol.getAddress()));
-      CIEInfo.LSDAPointerEncoding = LSDAPointerEncoding;
       break;
     }
     case 'P': {
@@ -347,8 +341,7 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B,
       if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding))
         return Err;
       if (PersonalityPointerEncoding !=
-          (dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
-           dwarf::DW_EH_PE_sdata4))
+          (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4))
         return make_error<JITLinkError>(
             "Unspported personality pointer "
             "encoding " +
@@ -363,12 +356,12 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B,
       uint8_t FDEPointerEncoding;
       if (auto Err = RecordReader.readInteger(FDEPointerEncoding))
         return Err;
-      if (!isSupportedPointerEncoding(FDEPointerEncoding))
+      if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr))
         return make_error<JITLinkError>(
-            "Unsupported FDE pointer encoding " +
+            "Unsupported FDE address pointer "
+            "encoding " +
             formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " +
             formatv("{0:x16}", CIESymbol.getAddress()));
-      CIEInfo.FDEPointerEncoding = FDEPointerEncoding;
       break;
     }
     default:
@@ -452,13 +445,11 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
     JITTargetAddress PCBeginFieldOffset = RecordReader.getOffset();
     auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset);
     if (PCEdgeItr == BlockEdges.end()) {
-      auto PCBeginPtrInfo =
-          readEncodedPointer(CIEInfo->FDEPointerEncoding,
-                             RecordAddress + PCBeginFieldOffset, RecordReader);
-      if (!PCBeginPtrInfo)
-        return PCBeginPtrInfo.takeError();
-      JITTargetAddress PCBegin = PCBeginPtrInfo->first;
-      Edge::Kind PCBeginEdgeKind = PCBeginPtrInfo->second;
+      auto PCBeginDelta = readAbsolutePointer(PC.G, RecordReader);
+      if (!PCBeginDelta)
+        return PCBeginDelta.takeError();
+      JITTargetAddress PCBegin =
+          RecordAddress + PCBeginFieldOffset + *PCBeginDelta;
       LLVM_DEBUG({
         dbgs() << "        Adding edge at "
                << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset)
@@ -467,8 +458,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
       auto PCBeginSym = getOrCreateSymbol(PC, PCBegin);
       if (!PCBeginSym)
         return PCBeginSym.takeError();
-      B.addEdge(PCBeginEdgeKind, RecordOffset + PCBeginFieldOffset, *PCBeginSym,
-                0);
+      B.addEdge(Delta64, RecordOffset + PCBeginFieldOffset, *PCBeginSym, 0);
       PCBeginBlock = &PCBeginSym->getBlock();
     } else {
       auto &EI = PCEdgeItr->second;
@@ -489,42 +479,38 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
                                         " points at external block");
       }
       PCBeginBlock = &EI.Target->getBlock();
-      if (auto Err = RecordReader.skip(
-              getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding)))
+      if (auto Err = RecordReader.skip(PC.G.getPointerSize()))
         return Err;
     }
 
     // Add a keep-alive edge from the FDE target to the FDE to ensure that the
     // FDE is kept alive if its target is.
     assert(PCBeginBlock && "PC-begin block not recorded");
-    LLVM_DEBUG({
-      dbgs() << "        Adding keep-alive edge from target at "
-             << formatv("{0:x16}", PCBeginBlock->getAddress()) << " to FDE at "
-             << formatv("{0:x16}", RecordAddress) << "\n";
-    });
     PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0);
   }
 
   // Skip over the PC range size field.
-  if (auto Err = RecordReader.skip(
-          getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding)))
+  if (auto Err = RecordReader.skip(PC.G.getPointerSize()))
     return Err;
 
   if (CIEInfo->FDEsHaveLSDAField) {
     uint64_t AugmentationDataSize;
     if (auto Err = RecordReader.readULEB128(AugmentationDataSize))
       return Err;
+    if (AugmentationDataSize != PC.G.getPointerSize())
+      return make_error<JITLinkError>(
+          "Unexpected FDE augmentation data size (expected " +
+          Twine(PC.G.getPointerSize()) + ", got " +
+          Twine(AugmentationDataSize) + ") for FDE at " +
+          formatv("{0:x16}", RecordAddress));
 
     JITTargetAddress LSDAFieldOffset = RecordReader.getOffset();
     auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset);
     if (LSDAEdgeItr == BlockEdges.end()) {
-      auto LSDAPointerInfo =
-          readEncodedPointer(CIEInfo->LSDAPointerEncoding,
-                             RecordAddress + LSDAFieldOffset, RecordReader);
-      if (!LSDAPointerInfo)
-        return LSDAPointerInfo.takeError();
-      JITTargetAddress LSDA = LSDAPointerInfo->first;
-      Edge::Kind LSDAEdgeKind = LSDAPointerInfo->second;
+      auto LSDADelta = readAbsolutePointer(PC.G, RecordReader);
+      if (!LSDADelta)
+        return LSDADelta.takeError();
+      JITTargetAddress LSDA = RecordAddress + LSDAFieldOffset + *LSDADelta;
       auto LSDASym = getOrCreateSymbol(PC, LSDA);
       if (!LSDASym)
         return LSDASym.takeError();
@@ -533,7 +519,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
                << formatv("{0:x16}", RecordAddress + LSDAFieldOffset)
                << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n";
       });
-      B.addEdge(LSDAEdgeKind, RecordOffset + LSDAFieldOffset, *LSDASym, 0);
+      B.addEdge(Delta64, RecordOffset + LSDAFieldOffset, *LSDASym, 0);
     } else {
       LLVM_DEBUG({
         auto &EI = LSDAEdgeItr->second;
@@ -544,7 +530,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
           dbgs() << " + " << formatv("{0:x16}", EI.Addend);
         dbgs() << "\n";
       });
-      if (auto Err = RecordReader.skip(AugmentationDataSize))
+      if (auto Err = RecordReader.skip(PC.G.getPointerSize()))
         return Err;
     }
   } else {
@@ -595,110 +581,23 @@ EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) {
   return std::move(AugInfo);
 }
 
-bool EHFrameEdgeFixer::isSupportedPointerEncoding(uint8_t PointerEncoding) {
-  using namespace dwarf;
-
-  // We only support PC-rel for now.
-  if ((PointerEncoding & 0x70) != DW_EH_PE_pcrel)
-    return false;
-
-  // readEncodedPointer does not handle indirect.
-  if (PointerEncoding & DW_EH_PE_indirect)
-    return false;
-
-  // Supported datatypes.
-  switch (PointerEncoding & 0xf) {
-  case DW_EH_PE_absptr:
-  case DW_EH_PE_udata4:
-  case DW_EH_PE_udata8:
-  case DW_EH_PE_sdata4:
-  case DW_EH_PE_sdata8:
-    return true;
-  }
-
-  return false;
-}
-
-unsigned EHFrameEdgeFixer::getPointerEncodingDataSize(uint8_t PointerEncoding) {
-  using namespace dwarf;
-
-  assert(isSupportedPointerEncoding(PointerEncoding) &&
-         "Unsupported pointer encoding");
-  switch (PointerEncoding & 0xf) {
-  case DW_EH_PE_absptr:
-    return PointerSize;
-  case DW_EH_PE_udata4:
-  case DW_EH_PE_sdata4:
-    return 4;
-  case DW_EH_PE_udata8:
-  case DW_EH_PE_sdata8:
-    return 8;
-  default:
-    llvm_unreachable("Unsupported encoding");
-  }
-}
-
-Expected<std::pair<JITTargetAddress, Edge::Kind>>
-EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding,
-                                     JITTargetAddress PointerFieldAddress,
-                                     BinaryStreamReader &RecordReader) {
+Expected<JITTargetAddress>
+EHFrameEdgeFixer::readAbsolutePointer(LinkGraph &G,
+                                      BinaryStreamReader &RecordReader) {
   static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t),
                 "Result must be able to hold a uint64_t");
-  assert(isSupportedPointerEncoding(PointerEncoding) &&
-         "Unsupported pointer encoding");
-
-  using namespace dwarf;
-
-  // Isolate data type, remap absptr to udata4 or udata8. This relies on us
-  // having verified that the graph uses 32-bit or 64-bit pointers only at the
-  // start of this pass.
-  uint8_t EffectiveType = PointerEncoding & 0xf;
-  if (EffectiveType == DW_EH_PE_absptr)
-    EffectiveType = (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4;
-
   JITTargetAddress Addr;
-  Edge::Kind PointerEdgeKind;
-  switch (EffectiveType) {
-  case DW_EH_PE_udata4: {
-    uint32_t Val;
-    if (auto Err = RecordReader.readInteger(Val))
+  if (G.getPointerSize() == 8) {
+    if (auto Err = RecordReader.readInteger(Addr))
       return std::move(Err);
-    Addr = PointerFieldAddress + Val;
-    PointerEdgeKind = Delta32;
-    break;
-  }
-  case DW_EH_PE_udata8: {
-    uint64_t Val;
-    if (auto Err = RecordReader.readInteger(Val))
+  } else if (G.getPointerSize() == 4) {
+    uint32_t Addr32;
+    if (auto Err = RecordReader.readInteger(Addr32))
       return std::move(Err);
-    Addr = PointerFieldAddress + Val;
-    PointerEdgeKind = Delta64;
-    break;
-  }
-  case DW_EH_PE_sdata4: {
-    int32_t Val;
-    if (auto Err = RecordReader.readInteger(Val))
-      return std::move(Err);
-    Addr = PointerFieldAddress + Val;
-    PointerEdgeKind = Delta32;
-    break;
-  }
-  case DW_EH_PE_sdata8: {
-    int64_t Val;
-    if (auto Err = RecordReader.readInteger(Val))
-      return std::move(Err);
-    Addr = PointerFieldAddress + Val;
-    PointerEdgeKind = Delta64;
-    break;
-  }
-  }
-
-  if (PointerEdgeKind == Edge::Invalid)
-    return make_error<JITLinkError>(
-        "Unspported edge kind for encoded pointer at " +
-        formatv("{0:x}", PointerFieldAddress));
-
-  return std::make_pair(Addr, Delta64);
+    Addr = Addr32;
+  } else
+    llvm_unreachable("Pointer size is not 32-bit or 64-bit");
+  return Addr;
 }
 
 Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC,
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
index 5e68e72ba18d..83f27a285998 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
@@ -40,8 +40,7 @@ class EHFrameSplitter {
 /// edges.
 class EHFrameEdgeFixer {
 public:
-  EHFrameEdgeFixer(StringRef EHFrameSectionName, unsigned PointerSize,
-                   Edge::Kind Delta64, Edge::Kind Delta32,
+  EHFrameEdgeFixer(StringRef EHFrameSectionName, Edge::Kind Delta64,
                    Edge::Kind NegDelta32);
   Error operator()(LinkGraph &G);
 
@@ -58,8 +57,6 @@ class EHFrameEdgeFixer {
     CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {}
     Symbol *CIESymbol = nullptr;
     bool FDEsHaveLSDAField = false;
-    uint8_t FDEPointerEncoding = 0;
-    uint8_t LSDAPointerEncoding = 0;
   };
 
   struct EdgeTarget {
@@ -99,20 +96,12 @@ class EHFrameEdgeFixer {
 
   Expected<AugmentationInfo>
   parseAugmentationString(BinaryStreamReader &RecordReader);
-
-  static bool isSupportedPointerEncoding(uint8_t PointerEncoding);
-  unsigned getPointerEncodingDataSize(uint8_t PointerEncoding);
-  Expected<std::pair<JITTargetAddress, Edge::Kind>>
-  readEncodedPointer(uint8_t PointerEncoding,
-                     JITTargetAddress PointerFieldAddress,
-                     BinaryStreamReader &RecordReader);
-
+  Expected<JITTargetAddress>
+  readAbsolutePointer(LinkGraph &G, BinaryStreamReader &RecordReader);
   Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, JITTargetAddress Addr);
 
   StringRef EHFrameSectionName;
-  unsigned PointerSize;
   Edge::Kind Delta64;
-  Edge::Kind Delta32;
   Edge::Kind NegDelta32;
 };
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index 2a92f28030ce..0ca2130150a6 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -11,14 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
+#include "BasicGOTAndStubsBuilder.h"
+#include "JITLinkGeneric.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Endian.h"
 
-#include "BasicGOTAndStubsBuilder.h"
-#include "EHFrameSupportImpl.h"
-#include "JITLinkGeneric.h"
-
 #define DEBUG_TYPE "jitlink"
 
 using namespace llvm;
@@ -240,8 +238,6 @@ class ELFLinkGraphBuilder_x86_64 {
     switch (Type) {
     case ELF::R_X86_64_PC32:
       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
-    case ELF::R_X86_64_PC64:
-      return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
     case ELF::R_X86_64_64:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
     case ELF::R_X86_64_GOTPCREL:
@@ -408,6 +404,9 @@ class ELFLinkGraphBuilder_x86_64 {
       LLVM_DEBUG({
         dbgs() << "Adding relocations from section " << *RelSectName << "\n";
       });
+      // Deal with .eh_frame later
+      if (*RelSectName == StringRef(".rela.eh_frame"))
+        continue;
 
       auto UpdateSection = Obj.getSection(SecRef.sh_info);
       if (!UpdateSection)
@@ -735,11 +734,6 @@ class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
       *(ulittle64_t *)FixupPtr = Value;
       break;
     }
-    case ELFX86RelocationKind::Delta64: {
-      int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
-      *(little64_t *)FixupPtr = Value;
-      break;
-    }
     }
     return Error::success();
   }
@@ -766,28 +760,21 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
                      std::unique_ptr<JITLinkContext> Ctx) {
   PassConfiguration Config;
 
-  if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
-
-    Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame"));
-    Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
-        ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32));
-
-    // Construct a JITLinker and run the link function.
-    // Add a mark-live pass.
-    if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
-      Config.PrePrunePasses.push_back(std::move(MarkLive));
-    else
-      Config.PrePrunePasses.push_back(markAllSymbolsLive);
+  // Construct a JITLinker and run the link function.
+  // Add a mark-live pass.
+  if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
+    Config.PrePrunePasses.push_back(std::move(MarkLive));
+  else
+    Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
-    // Add an in-place GOT/Stubs pass.
-    Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
-      ELF_x86_64_GOTAndStubsBuilder(G).run();
-      return Error::success();
-    });
+  // Add an in-place GOT/Stubs pass.
+  Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
+    ELF_x86_64_GOTAndStubsBuilder(G).run();
+    return Error::success();
+  });
 
-    // Add GOT/Stubs optimizer pass.
-    Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs);
-  }
+  // Add GOT/Stubs optimizer pass.
+  Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs);
 
   if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config))
     return Ctx->notifyFailed(std::move(Err));
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
index 26e6859de91d..3555d66ace35 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
@@ -16,10 +16,10 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
-#include "llvm/Object/MachO.h"
 
 #include "EHFrameSupportImpl.h"
 #include "JITLinkGeneric.h"
+#include "llvm/Object/MachO.h"
 
 #include <list>
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index bde4a19e71ba..24559cc7e772 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -669,8 +669,8 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
   if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
     // Add eh-frame passses.
     Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame"));
-    Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
-        "__eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32));
+    Config.PrePrunePasses.push_back(
+        EHFrameEdgeFixer("__eh_frame", Delta64, NegDelta32));
 
     // Add a mark-live pass.
     if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 8d960770313b..2b12e656c45e 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5587,18 +5587,17 @@ void Verifier::verifyNoAliasScopeDecl() {
     } while (ItNext != NoAliasScopeDecls.end() &&
              GetScope(*ItNext) == CurScope);
 
-    // [ItCurrent, ItNext[ represents the declarations for the same scope.
-    // Ensure they are not dominating each other
-    for (auto *I : llvm::make_range(ItCurrent, ItNext)) {
-      for (auto *J : llvm::make_range(ItCurrent, ItNext)) {
-        if (I != J) {
-          Assert(!DT.dominates(I, J),
-                 "llvm.experimental.noalias.scope.decl dominates another one "
-                 "with the same scope",
-                 I);
-        }
-      }
-    }
+    // [ItCurrent, ItNext) represents the declarations for the same scope.
+    // Ensure they are not dominating each other.. but only if it is not too
+    // expensive.
+    if (ItNext - ItCurrent < 32)
+      for (auto *I : llvm::make_range(ItCurrent, ItNext))
+        for (auto *J : llvm::make_range(ItCurrent, ItNext))
+          if (I != J)
+            Assert(!DT.dominates(I, J),
+                   "llvm.experimental.noalias.scope.decl dominates another one "
+                   "with the same scope",
+                   I);
     ItCurrent = ItNext;
   }
 }
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 752a037d61b1..5f1b031fe1d8 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -124,6 +124,11 @@ class ContiguousBlobAccumulator {
     if (checkLimit(sizeof(T)))
       support::endian::write<T>(OS, Val, E);
   }
+
+  void updateDataAt(uint64_t Pos, void *Data, size_t Size) {
+    assert(Pos >= InitialOffset && Pos + Size <= getOffset());
+    memcpy(&Buf[Pos - InitialOffset], Data, Size);
+  }
 };
 
 // Used to keep track of section and symbol names, so that in the YAML file
@@ -224,7 +229,7 @@ template <class ELFT> class ELFState {
                    ArrayRef<typename ELFT::Shdr> SHeaders);
 
   void finalizeStrings();
-  void writeELFHeader(raw_ostream &OS, Optional<uint64_t> SHOff);
+  void writeELFHeader(raw_ostream &OS);
   void writeSectionContent(Elf_Shdr &SHeader,
                            const ELFYAML::NoBitsSection &Section,
                            ContiguousBlobAccumulator &CBA);
@@ -330,12 +335,22 @@ ELFState<ELFT>::ELFState(ELFYAML::Object &D, yaml::ErrorHandler EH)
         std::make_unique<ELFYAML::Section>(
             ELFYAML::Chunk::ChunkKind::RawContent, /*IsImplicit=*/true));
 
-  // We add a technical suffix for each unnamed section/fill. It does not affect
-  // the output, but allows us to map them by name in the code and report better
-  // error messages.
   StringSet<> DocSections;
+  ELFYAML::SectionHeaderTable *SecHdrTable = nullptr;
   for (size_t I = 0; I < Doc.Chunks.size(); ++I) {
     const std::unique_ptr<ELFYAML::Chunk> &C = Doc.Chunks[I];
+
+    // We might have an explicit section header table declaration.
+    if (auto S = dyn_cast<ELFYAML::SectionHeaderTable>(C.get())) {
+      if (SecHdrTable)
+        reportError("multiple section header tables are not allowed");
+      SecHdrTable = S;
+      continue;
+    }
+
+    // We add a technical suffix for each unnamed section/fill. It does not
+    // affect the output, but allows us to map them by name in the code and
+    // report better error messages.
     if (C->Name.empty()) {
       std::string NewName = ELFYAML::appendUniqueSuffix(
           /*Name=*/"", "index " + Twine(I));
@@ -359,7 +374,7 @@ ELFState<ELFT>::ELFState(ELFYAML::Object &D, yaml::ErrorHandler EH)
       ImplicitSections.push_back(StringRef(SecName).copy(StringAlloc));
     }
   ImplicitSections.insert(ImplicitSections.end(), {".strtab"});
-  if (!Doc.SectionHeaders || !Doc.SectionHeaders->NoHeaders.getValueOr(false))
+  if (!SecHdrTable || !SecHdrTable->NoHeaders.getValueOr(false))
     ImplicitSections.insert(ImplicitSections.end(), {".shstrtab"});
 
   // Insert placeholders for implicit sections that are not
@@ -371,12 +386,27 @@ ELFState<ELFT>::ELFState(ELFYAML::Object &D, yaml::ErrorHandler EH)
     std::unique_ptr<ELFYAML::Chunk> Sec = std::make_unique<ELFYAML::Section>(
         ELFYAML::Chunk::ChunkKind::RawContent, true /*IsImplicit*/);
     Sec->Name = SecName;
-    Doc.Chunks.push_back(std::move(Sec));
+
+    // When the section header table is explicitly defined at the end of the
+    // sections list, it is reasonable to assume that the user wants to reorder
+    // section headers, but still wants to place the section header table after
+    // all sections, like it normally happens. In this case we want to insert
+    // other implicit sections right before the section header table.
+    if (Doc.Chunks.back().get() == SecHdrTable)
+      Doc.Chunks.insert(Doc.Chunks.end() - 1, std::move(Sec));
+    else
+      Doc.Chunks.push_back(std::move(Sec));
   }
+
+  // Insert the section header table implicitly at the end, when it is not
+  // explicitly defined.
+  if (!SecHdrTable)
+    Doc.Chunks.push_back(
+        std::make_unique<ELFYAML::SectionHeaderTable>(/*IsImplicit=*/true));
 }
 
 template <class ELFT>
-void ELFState<ELFT>::writeELFHeader(raw_ostream &OS, Optional<uint64_t> SHOff) {
+void ELFState<ELFT>::writeELFHeader(raw_ostream &OS) {
   using namespace llvm::ELF;
 
   Elf_Ehdr Header;
@@ -426,29 +456,24 @@ void ELFState<ELFT>::writeELFHeader(raw_ostream &OS, Optional<uint64_t> SHOff) {
   Header.e_shentsize = Doc.Header.EShEntSize ? (uint16_t)*Doc.Header.EShEntSize
                                              : sizeof(Elf_Shdr);
 
+  const ELFYAML::SectionHeaderTable &SectionHeaders =
+      Doc.getSectionHeaderTable();
+
   if (Doc.Header.EShOff)
     Header.e_shoff = *Doc.Header.EShOff;
-  else if (SHOff)
-    Header.e_shoff = *SHOff;
+  else if (SectionHeaders.Offset)
+    Header.e_shoff = *SectionHeaders.Offset;
   else
     Header.e_shoff = 0;
 
   if (Doc.Header.EShNum)
     Header.e_shnum = *Doc.Header.EShNum;
-  else if (!Doc.SectionHeaders ||
-           (Doc.SectionHeaders->NoHeaders && !*Doc.SectionHeaders->NoHeaders))
-    Header.e_shnum = Doc.getSections().size();
-  else if (!SHOff)
-    Header.e_shnum = 0;
   else
-    Header.e_shnum =
-        (Doc.SectionHeaders->Sections ? Doc.SectionHeaders->Sections->size()
-                                      : 0) +
-        /*Null section*/ 1;
+    Header.e_shnum = SectionHeaders.getNumHeaders(Doc.getSections().size());
 
   if (Doc.Header.EShStrNdx)
     Header.e_shstrndx = *Doc.Header.EShStrNdx;
-  else if (SHOff && !ExcludedSectionHeaders.count(".shstrtab"))
+  else if (SectionHeaders.Offset && !ExcludedSectionHeaders.count(".shstrtab"))
     Header.e_shstrndx = SN2I.get(".shstrtab");
   else
     Header.e_shstrndx = 0;
@@ -520,14 +545,16 @@ unsigned ELFState<ELFT>::toSectionIndex(StringRef S, StringRef LocSec,
     return 0;
   }
 
-  if (!Doc.SectionHeaders || (Doc.SectionHeaders->NoHeaders &&
-                              !Doc.SectionHeaders->NoHeaders.getValue()))
+  const ELFYAML::SectionHeaderTable &SectionHeaders =
+      Doc.getSectionHeaderTable();
+  if (SectionHeaders.IsImplicit ||
+      (SectionHeaders.NoHeaders && !SectionHeaders.NoHeaders.getValue()))
     return Index;
 
-  assert(!Doc.SectionHeaders->NoHeaders.getValueOr(false) ||
-         !Doc.SectionHeaders->Sections);
+  assert(!SectionHeaders.NoHeaders.getValueOr(false) ||
+         !SectionHeaders.Sections);
   size_t FirstExcluded =
-      Doc.SectionHeaders->Sections ? Doc.SectionHeaders->Sections->size() : 0;
+      SectionHeaders.Sections ? SectionHeaders.Sections->size() : 0;
   if (Index >= FirstExcluded) {
     if (LocSym.empty())
       reportError("unable to link '" + LocSec + "' to excluded section '" + S +
@@ -670,8 +697,27 @@ void ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders,
       continue;
     }
 
+    if (ELFYAML::SectionHeaderTable *S =
+            dyn_cast<ELFYAML::SectionHeaderTable>(D.get())) {
+      if (S->NoHeaders.getValueOr(false))
+        continue;
+
+      if (!S->Offset)
+        S->Offset = alignToOffset(CBA, sizeof(typename ELFT::uint),
+                                  /*Offset=*/None);
+      else
+        S->Offset = alignToOffset(CBA, /*Align=*/1, S->Offset);
+
+      uint64_t Size = S->getNumHeaders(SHeaders.size()) * sizeof(Elf_Shdr);
+      // The full section header information might be not available here, so
+      // fill the space with zeroes as a placeholder.
+      CBA.writeZeros(Size);
+      LocationCounter += Size;
+      continue;
+    }
+
     ELFYAML::Section *Sec = cast<ELFYAML::Section>(D.get());
-    bool IsFirstUndefSection = D == Doc.Chunks.front();
+    bool IsFirstUndefSection = Sec == Doc.getSections().front();
     if (IsFirstUndefSection && Sec->IsImplicit)
       continue;
 
@@ -1742,7 +1788,9 @@ void ELFState<ELFT>::writeFill(ELFYAML::Fill &Fill,
 
 template <class ELFT>
 DenseMap<StringRef, size_t> ELFState<ELFT>::buildSectionHeaderReorderMap() {
-  if (!Doc.SectionHeaders || Doc.SectionHeaders->NoHeaders)
+  const ELFYAML::SectionHeaderTable &SectionHeaders =
+      Doc.getSectionHeaderTable();
+  if (SectionHeaders.IsImplicit || SectionHeaders.NoHeaders)
     return DenseMap<StringRef, size_t>();
 
   DenseMap<StringRef, size_t> Ret;
@@ -1756,12 +1804,12 @@ DenseMap<StringRef, size_t> ELFState<ELFT>::buildSectionHeaderReorderMap() {
     Seen.insert(Hdr.Name);
   };
 
-  if (Doc.SectionHeaders->Sections)
-    for (const ELFYAML::SectionHeader &Hdr : *Doc.SectionHeaders->Sections)
+  if (SectionHeaders.Sections)
+    for (const ELFYAML::SectionHeader &Hdr : *SectionHeaders.Sections)
       AddSection(Hdr);
 
-  if (Doc.SectionHeaders->Excluded)
-    for (const ELFYAML::SectionHeader &Hdr : *Doc.SectionHeaders->Excluded)
+  if (SectionHeaders.Excluded)
+    for (const ELFYAML::SectionHeader &Hdr : *SectionHeaders.Excluded)
       AddSection(Hdr);
 
   for (const ELFYAML::Section *S : Doc.getSections()) {
@@ -1790,17 +1838,17 @@ template <class ELFT> void ELFState<ELFT>::buildSectionIndex() {
 
   // Build excluded section headers map.
   std::vector<ELFYAML::Section *> Sections = Doc.getSections();
-  if (Doc.SectionHeaders) {
-    if (Doc.SectionHeaders->Excluded)
-      for (const ELFYAML::SectionHeader &Hdr : *Doc.SectionHeaders->Excluded)
-        if (!ExcludedSectionHeaders.insert(Hdr.Name).second)
-          llvm_unreachable("buildSectionIndex() failed");
-
-    if (Doc.SectionHeaders->NoHeaders.getValueOr(false))
-      for (const ELFYAML::Section *S : Sections)
-        if (!ExcludedSectionHeaders.insert(S->Name).second)
-          llvm_unreachable("buildSectionIndex() failed");
-  }
+  const ELFYAML::SectionHeaderTable &SectionHeaders =
+      Doc.getSectionHeaderTable();
+  if (SectionHeaders.Excluded)
+    for (const ELFYAML::SectionHeader &Hdr : *SectionHeaders.Excluded)
+      if (!ExcludedSectionHeaders.insert(Hdr.Name).second)
+        llvm_unreachable("buildSectionIndex() failed");
+
+  if (SectionHeaders.NoHeaders.getValueOr(false))
+    for (const ELFYAML::Section *S : Sections)
+      if (!ExcludedSectionHeaders.insert(S->Name).second)
+        llvm_unreachable("buildSectionIndex() failed");
 
   size_t SecNdx = -1;
   for (const ELFYAML::Section *S : Sections) {
@@ -1903,17 +1951,7 @@ bool ELFState<ELFT>::writeELF(raw_ostream &OS, ELFYAML::Object &Doc,
   // Now we can decide segment offsets.
   State.setProgramHeaderLayout(PHeaders, SHeaders);
 
-  // If needed, align the start of the section header table, which is written
-  // after all section data.
-  const bool HasSectionHeaders =
-      !Doc.SectionHeaders || !Doc.SectionHeaders->NoHeaders.getValueOr(false);
-  Optional<uint64_t> SHOff;
-  if (HasSectionHeaders)
-    SHOff = State.alignToOffset(CBA, sizeof(typename ELFT::uint),
-                                /*Offset=*/None);
-  bool ReachedLimit = SHOff.getValueOr(CBA.getOffset()) +
-                          arrayDataSize(makeArrayRef(SHeaders)) >
-                      MaxSize;
+  bool ReachedLimit = CBA.getOffset() > MaxSize;
   if (Error E = CBA.takeLimitError()) {
     // We report a custom error message instead below.
     consumeError(std::move(E));
@@ -1928,11 +1966,15 @@ bool ELFState<ELFT>::writeELF(raw_ostream &OS, ELFYAML::Object &Doc,
   if (State.HasError)
     return false;
 
-  State.writeELFHeader(OS, SHOff);
+  State.writeELFHeader(OS);
   writeArrayData(OS, makeArrayRef(PHeaders));
+
+  const ELFYAML::SectionHeaderTable &SHT = Doc.getSectionHeaderTable();
+  if (!SHT.NoHeaders.getValueOr(false))
+    CBA.updateDataAt(*SHT.Offset, SHeaders.data(),
+                     SHT.getNumHeaders(SHeaders.size()) * sizeof(Elf_Shdr));
+
   CBA.writeBlobToStream(OS);
-  if (HasSectionHeaders)
-    writeArrayData(OS, makeArrayRef(SHeaders));
   return true;
 }
 
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 84022cf6b416..05d30577812b 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -33,6 +33,8 @@ unsigned Object::getMachine() const {
     return *Header.Machine;
   return llvm::ELF::EM_NONE;
 }
+
+constexpr StringRef SectionHeaderTable::TypeStr;
 } // namespace ELFYAML
 
 namespace yaml {
@@ -860,23 +862,6 @@ void MappingTraits<ELFYAML::SectionHeader>::mapping(
   IO.mapRequired("Name", SHdr.Name);
 }
 
-void MappingTraits<ELFYAML::SectionHeaderTable>::mapping(
-    IO &IO, ELFYAML::SectionHeaderTable &SectionHeader) {
-  IO.mapOptional("Sections", SectionHeader.Sections);
-  IO.mapOptional("Excluded", SectionHeader.Excluded);
-  IO.mapOptional("NoHeaders", SectionHeader.NoHeaders);
-}
-
-std::string MappingTraits<ELFYAML::SectionHeaderTable>::validate(
-    IO &IO, ELFYAML::SectionHeaderTable &SecHdrTable) {
-  if (SecHdrTable.NoHeaders && (SecHdrTable.Sections || SecHdrTable.Excluded))
-    return "NoHeaders can't be used together with Sections/Excluded";
-  if (!SecHdrTable.NoHeaders && !SecHdrTable.Sections && !SecHdrTable.Excluded)
-    return "SectionHeaderTable can't be empty. Use 'NoHeaders' key to drop the "
-           "section header table";
-  return "";
-}
-
 void MappingTraits<ELFYAML::FileHeader>::mapping(IO &IO,
                                                  ELFYAML::FileHeader &FileHdr) {
   IO.mapRequired("Class", FileHdr.Class);
@@ -1252,6 +1237,14 @@ static void fillMapping(IO &IO, ELFYAML::Fill &Fill) {
   IO.mapRequired("Size", Fill.Size);
 }
 
+static void sectionHeaderTableMapping(IO &IO,
+                                      ELFYAML::SectionHeaderTable &SHT) {
+  IO.mapOptional("Offset", SHT.Offset);
+  IO.mapOptional("Sections", SHT.Sections);
+  IO.mapOptional("Excluded", SHT.Excluded);
+  IO.mapOptional("NoHeaders", SHT.NoHeaders);
+}
+
 static void sectionMapping(IO &IO, ELFYAML::LinkerOptionsSection &Section) {
   commonSectionMapping(IO, Section);
   IO.mapOptional("Options", Section.Options);
@@ -1304,22 +1297,48 @@ static StringRef getStringValue(IO &IO, const char *Key) {
   return Val;
 }
 
+static void setStringValue(IO &IO, const char *Key, StringRef Val) {
+  IO.mapRequired(Key, Val);
+}
+
+static bool isInteger(StringRef Val) {
+  APInt Tmp;
+  return !Val.getAsInteger(0, Tmp);
+}
+
 void MappingTraits<std::unique_ptr<ELFYAML::Chunk>>::mapping(
     IO &IO, std::unique_ptr<ELFYAML::Chunk> &Section) {
   ELFYAML::ELF_SHT Type;
+  StringRef TypeStr;
   if (IO.outputting()) {
-    Type = cast<ELFYAML::Section>(Section.get())->Type;
+    if (auto *S = dyn_cast<ELFYAML::Section>(Section.get()))
+      Type = S->Type;
+    else if (auto *SHT = dyn_cast<ELFYAML::SectionHeaderTable>(Section.get()))
+      TypeStr = SHT->TypeStr;
   } else {
     // When the Type string does not have a "SHT_" prefix, we know it is not a
-    // description of a regular ELF output section. Currently, we have one
-    // special type named "Fill". See comments for Fill.
-    if (getStringValue(IO, "Type") == "Fill") {
-      Section.reset(new ELFYAML::Fill());
-      fillMapping(IO, *cast<ELFYAML::Fill>(Section.get()));
-      return;
-    }
+    // description of a regular ELF output section.
+    TypeStr = getStringValue(IO, "Type");
+    if (TypeStr.startswith("SHT_") || isInteger(TypeStr))
+      IO.mapRequired("Type", Type);
+  }
+
+  if (TypeStr == "Fill") {
+    assert(!IO.outputting()); // We don't dump fills currently.
+    Section.reset(new ELFYAML::Fill());
+    fillMapping(IO, *cast<ELFYAML::Fill>(Section.get()));
+    return;
+  }
 
-    IO.mapRequired("Type", Type);
+  if (TypeStr == ELFYAML::SectionHeaderTable::TypeStr) {
+    if (IO.outputting())
+      setStringValue(IO, "Type", TypeStr);
+    else
+      Section.reset(new ELFYAML::SectionHeaderTable(/*IsImplicit=*/false));
+
+    sectionHeaderTableMapping(
+        IO, *cast<ELFYAML::SectionHeaderTable>(Section.get()));
+    return;
   }
 
   const auto &Obj = *static_cast<ELFYAML::Object *>(IO.getContext());
@@ -1452,6 +1471,15 @@ std::string MappingTraits<std::unique_ptr<ELFYAML::Chunk>>::validate(
     return "";
   }
 
+  if (const auto *SHT = dyn_cast<ELFYAML::SectionHeaderTable>(C.get())) {
+    if (SHT->NoHeaders && (SHT->Sections || SHT->Excluded || SHT->Offset))
+      return "NoHeaders can't be used together with Offset/Sections/Excluded";
+    if (!SHT->NoHeaders && !SHT->Sections && !SHT->Excluded)
+      return "SectionHeaderTable can't be empty. Use 'NoHeaders' key to drop "
+             "the section header table";
+    return "";
+  }
+
   const ELFYAML::Section &Sec = *cast<ELFYAML::Section>(C.get());
   if (Sec.Size && Sec.Content &&
       (uint64_t)(*Sec.Size) < Sec.Content->binary_size())
@@ -1656,7 +1684,6 @@ void MappingTraits<ELFYAML::Object>::mapping(IO &IO, ELFYAML::Object &Object) {
   IO.mapOptional("Symbols", Object.Symbols);
   IO.mapOptional("DynamicSymbols", Object.DynamicSymbols);
   IO.mapOptional("DWARF", Object.DWARF);
-  IO.mapOptional("SectionHeaderTable", Object.SectionHeaders);
   if (Object.DWARF) {
     Object.DWARF->IsLittleEndian =
         Object.Header.Data == ELFYAML::ELF_ELFDATA(ELF::ELFDATA2LSB);
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 15c7130b24f3..762855207d2b 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -218,6 +218,10 @@ def FeatureArithmeticCbzFusion : SubtargetFeature<
     "arith-cbz-fusion", "HasArithmeticCbzFusion", "true",
     "CPU fuses arithmetic + cbz/cbnz operations">;
 
+def FeatureCmpBccFusion : SubtargetFeature<
+    "cmp-bcc-fusion", "HasCmpBccFusion", "true",
+    "CPU fuses cmp+bcc operations">;
+
 def FeatureFuseAddress : SubtargetFeature<
     "fuse-address", "HasFuseAddress", "true",
     "CPU fuses address generation and memory operations">;
@@ -615,6 +619,9 @@ def ProcA65     : SubtargetFeature<"a65", "ARMProcFamily", "CortexA65",
                                    FeatureDotProd,
                                    FeatureFPARMv8,
                                    FeatureFullFP16,
+                                   FeatureFuseAddress,
+                                   FeatureFuseAES,
+                                   FeatureFuseLiterals,
                                    FeatureNEON,
                                    FeatureRAS,
                                    FeatureRCPC,
@@ -627,6 +634,7 @@ def ProcA72     : SubtargetFeature<"a72", "ARMProcFamily", "CortexA72",
                                    FeatureCrypto,
                                    FeatureFPARMv8,
                                    FeatureFuseAES,
+                                   FeatureFuseLiterals,
                                    FeatureNEON,
                                    FeaturePerfMon
                                    ]>;
@@ -658,6 +666,7 @@ def ProcA76     : SubtargetFeature<"a76", "ARMProcFamily", "CortexA76",
                                    "Cortex-A76 ARM processors", [
                                     HasV8_2aOps,
                                     FeatureFPARMv8,
+                                    FeatureFuseAES,
                                     FeatureNEON,
                                     FeatureRCPC,
                                     FeatureCrypto,
@@ -669,7 +678,9 @@ def ProcA76     : SubtargetFeature<"a76", "ARMProcFamily", "CortexA76",
 def ProcA77     : SubtargetFeature<"a77", "ARMProcFamily", "CortexA77",
                                    "Cortex-A77 ARM processors", [
                                     HasV8_2aOps,
+                                    FeatureCmpBccFusion,
                                     FeatureFPARMv8,
+                                    FeatureFuseAES,
                                     FeatureNEON, FeatureRCPC,
                                     FeatureCrypto,
                                     FeatureFullFP16,
@@ -680,6 +691,7 @@ def ProcA78 : SubtargetFeature<"cortex-a78", "ARMProcFamily",
                                "CortexA78",
                                "Cortex-A78 ARM processors", [
                                HasV8_2aOps,
+                               FeatureCmpBccFusion,
                                FeatureCrypto,
                                FeatureFPARMv8,
                                FeatureFuseAES,
@@ -696,6 +708,7 @@ def ProcA78C : SubtargetFeature<"cortex-a78c", "ARMProcFamily",
                                 "CortexA78C",
                                 "Cortex-A78C ARM processors", [
                                 HasV8_2aOps,
+                                FeatureCmpBccFusion,
                                 FeatureCrypto,
                                 FeatureDotProd,
                                 FeatureFlagM,
@@ -723,6 +736,7 @@ def ProcR82 : SubtargetFeature<"cortex-r82", "ARMProcFamily",
 def ProcX1 : SubtargetFeature<"cortex-x1", "ARMProcFamily", "CortexX1",
                                   "Cortex-X1 ARM processors", [
                                   HasV8_2aOps,
+                                  FeatureCmpBccFusion,
                                   FeatureCrypto,
                                   FeatureFPARMv8,
                                   FeatureFuseAES,
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
index 9a2103579a6a..f3b8ef16d6f9 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
@@ -21,7 +21,7 @@ namespace {
 
 /// CMN, CMP, TST followed by Bcc
 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
-                                const MachineInstr &SecondMI) {
+                                const MachineInstr &SecondMI, bool CmpOnly) {
   if (SecondMI.getOpcode() != AArch64::Bcc)
     return false;
 
@@ -29,6 +29,13 @@ static bool isArithmeticBccPair(const MachineInstr *FirstMI,
   if (FirstMI == nullptr)
     return true;
 
+  // If we're in CmpOnly mode, we only fuse arithmetic instructions that
+  // discard their result.
+  if (CmpOnly && !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
+                   FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
+    return false;
+  }
+
   switch (FirstMI->getOpcode()) {
   case AArch64::ADDSWri:
   case AArch64::ADDSWrr:
@@ -380,8 +387,11 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
 
   // All checking functions assume that the 1st instr is a wildcard if it is
   // unspecified.
-  if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
-    return true;
+  if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
+    bool CmpOnly = !ST.hasArithmeticBccFusion();
+    if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
+      return true;
+  }
   if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
     return true;
   if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index b34911eafdd2..8fe2f125982f 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -221,6 +221,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool UseAlternateSExtLoadCVTF32Pattern = false;
   bool HasArithmeticBccFusion = false;
   bool HasArithmeticCbzFusion = false;
+  bool HasCmpBccFusion = false;
   bool HasFuseAddress = false;
   bool HasFuseAES = false;
   bool HasFuseArithmeticLogic = false;
@@ -377,6 +378,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   }
   bool hasArithmeticBccFusion() const { return HasArithmeticBccFusion; }
   bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; }
+  bool hasCmpBccFusion() const { return HasCmpBccFusion; }
   bool hasFuseAddress() const { return HasFuseAddress; }
   bool hasFuseAES() const { return HasFuseAES; }
   bool hasFuseArithmeticLogic() const { return HasFuseArithmeticLogic; }
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 5146271befff..035278135cef 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1297,7 +1297,8 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   parseNamedBit(const char *Name, OperandVector &Operands,
                 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
   OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
-                                             StringRef &Value);
+                                             StringRef &Value,
+                                             SMLoc &StringLoc);
 
   bool isModifier();
   bool isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
@@ -5099,11 +5100,15 @@ static void addOptionalImmOperand(
 }
 
 OperandMatchResultTy
-AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
+AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
+                                       StringRef &Value,
+                                       SMLoc &StringLoc) {
   if (!trySkipId(Prefix, AsmToken::Colon))
     return MatchOperand_NoMatch;
 
-  return parseId(Value) ? MatchOperand_Success : MatchOperand_ParseFail;
+  StringLoc = getLoc();
+  return parseId(Value, "expected an identifier") ? MatchOperand_Success
+                                                  : MatchOperand_ParseFail;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7523,7 +7528,8 @@ AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
   StringRef Value;
   OperandMatchResultTy res;
 
-  res = parseStringWithPrefix(Prefix, Value);
+  SMLoc StringLoc;
+  res = parseStringWithPrefix(Prefix, Value, StringLoc);
   if (res != MatchOperand_Success) {
     return res;
   }
@@ -7540,6 +7546,7 @@ AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
         .Default(0xffffffff);
 
   if (Int == 0xffffffff) {
+    Error(StringLoc, "invalid " + Twine(Prefix) + " value");
     return MatchOperand_ParseFail;
   }
 
@@ -7555,7 +7562,8 @@ AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
   StringRef Value;
   OperandMatchResultTy res;
 
-  res = parseStringWithPrefix("dst_unused", Value);
+  SMLoc StringLoc;
+  res = parseStringWithPrefix("dst_unused", Value, StringLoc);
   if (res != MatchOperand_Success) {
     return res;
   }
@@ -7568,6 +7576,7 @@ AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
         .Default(0xffffffff);
 
   if (Int == 0xffffffff) {
+    Error(StringLoc, "invalid dst_unused value");
     return MatchOperand_ParseFail;
   }
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index b4af0e81dfa7..11e7cf1113d2 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -1997,7 +1997,33 @@ bool RISCVAsmParser::parseDirectiveAttribute() {
     else
       return Error(ValueExprLoc, "bad arch string " + Arch);
 
+    // .attribute arch overrides the current architecture, so unset all
+    // currently enabled extensions
+    clearFeatureBits(RISCV::FeatureRV32E, "e");
+    clearFeatureBits(RISCV::FeatureStdExtM, "m");
+    clearFeatureBits(RISCV::FeatureStdExtA, "a");
+    clearFeatureBits(RISCV::FeatureStdExtF, "f");
+    clearFeatureBits(RISCV::FeatureStdExtD, "d");
+    clearFeatureBits(RISCV::FeatureStdExtC, "c");
+    clearFeatureBits(RISCV::FeatureStdExtB, "experimental-b");
+    clearFeatureBits(RISCV::FeatureStdExtV, "experimental-v");
+    clearFeatureBits(RISCV::FeatureExtZfh, "experimental-zfh");
+    clearFeatureBits(RISCV::FeatureExtZba, "experimental-zba");
+    clearFeatureBits(RISCV::FeatureExtZbb, "experimental-zbb");
+    clearFeatureBits(RISCV::FeatureExtZbc, "experimental-zbc");
+    clearFeatureBits(RISCV::FeatureExtZbe, "experimental-zbe");
+    clearFeatureBits(RISCV::FeatureExtZbf, "experimental-zbf");
+    clearFeatureBits(RISCV::FeatureExtZbm, "experimental-zbm");
+    clearFeatureBits(RISCV::FeatureExtZbp, "experimental-zbp");
+    clearFeatureBits(RISCV::FeatureExtZbproposedc, "experimental-zbproposedc");
+    clearFeatureBits(RISCV::FeatureExtZbr, "experimental-zbr");
+    clearFeatureBits(RISCV::FeatureExtZbs, "experimental-zbs");
+    clearFeatureBits(RISCV::FeatureExtZbt, "experimental-zbt");
+    clearFeatureBits(RISCV::FeatureExtZvamo, "experimental-zvamo");
+    clearFeatureBits(RISCV::FeatureStdExtZvlsseg, "experimental-zvlsseg");
+
     while (!Arch.empty()) {
+      bool DropFirst = true;
       if (Arch[0] == 'i')
         clearFeatureBits(RISCV::FeatureRV32E, "e");
       else if (Arch[0] == 'e')
@@ -2019,19 +2045,57 @@ bool RISCVAsmParser::parseDirectiveAttribute() {
         setFeatureBits(RISCV::FeatureStdExtD, "d");
       } else if (Arch[0] == 'c') {
         setFeatureBits(RISCV::FeatureStdExtC, "c");
+      } else if (Arch[0] == 'b') {
+        setFeatureBits(RISCV::FeatureStdExtB, "experimental-b");
+      } else if (Arch[0] == 'v') {
+        setFeatureBits(RISCV::FeatureStdExtV, "experimental-v");
+      } else if (Arch[0] == 's' || Arch[0] == 'x' || Arch[0] == 'z') {
+        StringRef Ext =
+            Arch.take_until([](char c) { return ::isdigit(c) || c == '_'; });
+        if (Ext == "zba")
+          setFeatureBits(RISCV::FeatureExtZba, "experimental-zba");
+        else if (Ext == "zbb")
+          setFeatureBits(RISCV::FeatureExtZbb, "experimental-zbb");
+        else if (Ext == "zbc")
+          setFeatureBits(RISCV::FeatureExtZbc, "experimental-zbc");
+        else if (Ext == "zbe")
+          setFeatureBits(RISCV::FeatureExtZbe, "experimental-zbe");
+        else if (Ext == "zbf")
+          setFeatureBits(RISCV::FeatureExtZbf, "experimental-zbf");
+        else if (Ext == "zbm")
+          setFeatureBits(RISCV::FeatureExtZbm, "experimental-zbm");
+        else if (Ext == "zbp")
+          setFeatureBits(RISCV::FeatureExtZbp, "experimental-zbp");
+        else if (Ext == "zbproposedc")
+          setFeatureBits(RISCV::FeatureExtZbproposedc,
+                         "experimental-zbproposedc");
+        else if (Ext == "zbr")
+          setFeatureBits(RISCV::FeatureExtZbr, "experimental-zbr");
+        else if (Ext == "zbs")
+          setFeatureBits(RISCV::FeatureExtZbs, "experimental-zbs");
+        else if (Ext == "zbt")
+          setFeatureBits(RISCV::FeatureExtZbt, "experimental-zbt");
+        else if (Ext == "zfh")
+          setFeatureBits(RISCV::FeatureExtZfh, "experimental-zfh");
+        else if (Ext == "zvamo")
+          setFeatureBits(RISCV::FeatureExtZvamo, "experimental-zvamo");
+        else if (Ext == "zvlsseg")
+          setFeatureBits(RISCV::FeatureStdExtZvlsseg, "experimental-zvlsseg");
+        else
+          return Error(ValueExprLoc, "bad arch string " + Ext);
+        Arch = Arch.drop_until([](char c) { return ::isdigit(c) || c == '_'; });
+        DropFirst = false;
       } else
         return Error(ValueExprLoc, "bad arch string " + Arch);
 
-      Arch = Arch.drop_front(1);
+      if (DropFirst)
+        Arch = Arch.drop_front(1);
       int major = 0;
       int minor = 0;
       Arch.consumeInteger(10, major);
       Arch.consume_front("p");
       Arch.consumeInteger(10, minor);
-      if (major != 0 || minor != 0) {
-        Arch = Arch.drop_until([](char c) { return c == '_' || c == '"'; });
-        Arch = Arch.drop_while([](char c) { return c == '_'; });
-      }
+      Arch = Arch.drop_while([](char c) { return c == '_'; });
     }
   }
 
@@ -2059,6 +2123,38 @@ bool RISCVAsmParser::parseDirectiveAttribute() {
         formalArchStr = (Twine(formalArchStr) + "_d2p0").str();
       if (getFeatureBits(RISCV::FeatureStdExtC))
         formalArchStr = (Twine(formalArchStr) + "_c2p0").str();
+      if (getFeatureBits(RISCV::FeatureStdExtB))
+        formalArchStr = (Twine(formalArchStr) + "_b0p93").str();
+      if (getFeatureBits(RISCV::FeatureStdExtV))
+        formalArchStr = (Twine(formalArchStr) + "_v0p9").str();
+      if (getFeatureBits(RISCV::FeatureExtZfh))
+        formalArchStr = (Twine(formalArchStr) + "_zfh0p1").str();
+      if (getFeatureBits(RISCV::FeatureExtZba))
+        formalArchStr = (Twine(formalArchStr) + "_zba0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbb))
+        formalArchStr = (Twine(formalArchStr) + "_zbb0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbc))
+        formalArchStr = (Twine(formalArchStr) + "_zbc0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbe))
+        formalArchStr = (Twine(formalArchStr) + "_zbe0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbf))
+        formalArchStr = (Twine(formalArchStr) + "_zbf0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbm))
+        formalArchStr = (Twine(formalArchStr) + "_zbm0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbp))
+        formalArchStr = (Twine(formalArchStr) + "_zbp0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbproposedc))
+        formalArchStr = (Twine(formalArchStr) + "_zbproposedc0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbr))
+        formalArchStr = (Twine(formalArchStr) + "_zbr0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbs))
+        formalArchStr = (Twine(formalArchStr) + "_zbs0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZbt))
+        formalArchStr = (Twine(formalArchStr) + "_zbt0p93").str();
+      if (getFeatureBits(RISCV::FeatureExtZvamo))
+        formalArchStr = (Twine(formalArchStr) + "_zvamo0p9").str();
+      if (getFeatureBits(RISCV::FeatureStdExtZvlsseg))
+        formalArchStr = (Twine(formalArchStr) + "_zvlsseg0p9").str();
 
       getTargetStreamer().emitTextAttribute(Tag, formalArchStr);
     }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
index 5683214c13fd..72206c7360e5 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
@@ -60,6 +60,38 @@ void RISCVTargetStreamer::emitTargetAttributes(const MCSubtargetInfo &STI) {
     Arch += "_d2p0";
   if (STI.hasFeature(RISCV::FeatureStdExtC))
     Arch += "_c2p0";
+  if (STI.hasFeature(RISCV::FeatureStdExtB))
+    Arch += "_b0p93";
+  if (STI.hasFeature(RISCV::FeatureStdExtV))
+    Arch += "_v0p9";
+  if (STI.hasFeature(RISCV::FeatureExtZfh))
+    Arch += "_zfh0p1";
+  if (STI.hasFeature(RISCV::FeatureExtZba))
+    Arch += "_zba0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbb))
+    Arch += "_zbb0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbc))
+    Arch += "_zbc0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbe))
+    Arch += "_zbe0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbf))
+    Arch += "_zbf0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbm))
+    Arch += "_zbm0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbp))
+    Arch += "_zbp0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbproposedc))
+    Arch += "_zbproposedc0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbr))
+    Arch += "_zbr0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbs))
+    Arch += "_zbs0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZbt))
+    Arch += "_zbt0p93";
+  if (STI.hasFeature(RISCV::FeatureExtZvamo))
+    Arch += "_zvamo0p9";
+  if (STI.hasFeature(RISCV::FeatureStdExtZvlsseg))
+    Arch += "_zvlsseg0p9";
 
   emitTextAttribute(RISCVAttrs::ARCH, Arch);
 }
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index a24e10318c78..0f9da5600a34 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -382,6 +382,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::UMIN, VT, Legal);
       setOperationAction(ISD::UMAX, VT, Legal);
 
+      setOperationAction(ISD::ROTL, VT, Expand);
+      setOperationAction(ISD::ROTR, VT, Expand);
+
       if (isTypeLegal(VT)) {
         // Custom-lower extensions and truncations from/to mask types.
         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
@@ -2010,6 +2013,8 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     // Comparing with 0 may allow us to fold into bnez/beqz.
     SDValue LHS = N->getOperand(0);
     SDValue RHS = N->getOperand(1);
+    if (LHS.getValueType().isScalableVector())
+      break;
     auto CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
     if (isOneConstant(RHS) && ISD::isIntEqualitySetCC(CC) &&
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 70203dacef09..fc19800eda79 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -21075,30 +21075,29 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
   assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!");
 
   if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
+    In = DAG.getBitcast(MVT::v8i32, In);
+
     // On AVX2, v4i64 -> v4i32 becomes VPERMD.
     if (Subtarget.hasInt256()) {
       static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
-      In = DAG.getBitcast(MVT::v8i32, In);
       In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
                          DAG.getIntPtrConstant(0, DL));
     }
 
-    SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
+    SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
                                DAG.getIntPtrConstant(0, DL));
-    SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
-                               DAG.getIntPtrConstant(2, DL));
-    OpLo = DAG.getBitcast(MVT::v4i32, OpLo);
-    OpHi = DAG.getBitcast(MVT::v4i32, OpHi);
+    SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
+                               DAG.getIntPtrConstant(4, DL));
     static const int ShufMask[] = {0, 2, 4, 6};
     return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);
   }
 
   if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
+    In = DAG.getBitcast(MVT::v32i8, In);
+
     // On AVX2, v8i32 -> v8i16 becomes PSHUFB.
     if (Subtarget.hasInt256()) {
-      In = DAG.getBitcast(MVT::v32i8, In);
-
       // The PSHUFB mask:
       static const int ShufMask1[] = { 0,  1,  4,  5,  8,  9, 12, 13,
                                       -1, -1, -1, -1, -1, -1, -1, -1,
@@ -21107,21 +21106,17 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
       In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
       In = DAG.getBitcast(MVT::v4i64, In);
 
-      static const int ShufMask2[] = {0,  2,  -1,  -1};
-      In = DAG.getVectorShuffle(MVT::v4i64, DL,  In, In, ShufMask2);
-      In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
-                       DAG.getIntPtrConstant(0, DL));
-      return DAG.getBitcast(VT, In);
+      static const int ShufMask2[] = {0, 2, -1, -1};
+      In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16,
+                         DAG.getBitcast(MVT::v16i16, In),
+                         DAG.getIntPtrConstant(0, DL));
     }
 
-    SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
+    SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, In,
                                DAG.getIntPtrConstant(0, DL));
-
-    SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
-                               DAG.getIntPtrConstant(4, DL));
-
-    OpLo = DAG.getBitcast(MVT::v16i8, OpLo);
-    OpHi = DAG.getBitcast(MVT::v16i8, OpHi);
+    SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, In,
+                               DAG.getIntPtrConstant(16, DL));
 
     // The PSHUFB mask:
     static const int ShufMask1[] = {0,  1,  4,  5,  8,  9, 12, 13,
@@ -33981,10 +33976,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     return emitPatchPoint(MI, BB);
 
   case TargetOpcode::PATCHABLE_EVENT_CALL:
-    return emitXRayCustomEvent(MI, BB);
-
   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
-    return emitXRayTypedEvent(MI, BB);
+    return BB;
 
   case X86::LCMPXCHG8B: {
     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
@@ -35359,8 +35352,6 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   // Handle 128/256-bit lane shuffles of 512-bit vectors.
   if (RootVT.is512BitVector() &&
       (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) {
-    MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);
-
     // If the upper subvectors are zeroable, then an extract+insert is more
     // optimal than using X86ISD::SHUF128. The insertion is free, even if it has
     // to zero the upper subvectors.
@@ -35369,12 +35360,11 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
         return SDValue(); // Nothing to do!
       assert(isInRange(BaseMask[0], 0, NumBaseMaskElts) &&
              "Unexpected lane shuffle");
-      Res = CanonicalizeShuffleInput(ShuffleVT, V1);
-      unsigned SubIdx = BaseMask[0] * (8 / NumBaseMaskElts);
+      Res = CanonicalizeShuffleInput(RootVT, V1);
+      unsigned SubIdx = BaseMask[0] * (NumRootElts / NumBaseMaskElts);
       bool UseZero = isAnyZero(BaseMask);
       Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
-      Res = widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
-      return DAG.getBitcast(RootVT, Res);
+      return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
     }
 
     // Narrow shuffle mask to v4x128.
@@ -35425,6 +35415,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     if (!isAnyZero(Mask) && !PreferPERMQ) {
       if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
         return SDValue(); // Nothing to do!
+      MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);
       if (SDValue V = MatchSHUF128(ShuffleVT, DL, Mask, V1, V2, DAG))
         return DAG.getBitcast(RootVT, V);
     }
@@ -35432,8 +35423,6 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
 
   // Handle 128-bit lane shuffles of 256-bit vectors.
   if (RootVT.is256BitVector() && NumBaseMaskElts == 2) {
-    MVT ShuffleVT = (FloatDomain ? MVT::v4f64 : MVT::v4i64);
-
     // If the upper half is zeroable, then an extract+insert is more optimal
     // than using X86ISD::VPERM2X128. The insertion is free, even if it has to
     // zero the upper half.
@@ -35441,11 +35430,10 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
         return SDValue(); // Nothing to do!
       assert(isInRange(BaseMask[0], 0, 2) && "Unexpected lane shuffle");
-      Res = CanonicalizeShuffleInput(ShuffleVT, V1);
-      Res = extract128BitVector(Res, BaseMask[0] * 2, DAG, DL);
-      Res = widenSubVector(Res, BaseMask[1] == SM_SentinelZero, Subtarget, DAG,
-                           DL, 256);
-      return DAG.getBitcast(RootVT, Res);
+      Res = CanonicalizeShuffleInput(RootVT, V1);
+      Res = extract128BitVector(Res, BaseMask[0] * (NumRootElts / 2), DAG, DL);
+      return widenSubVector(Res, BaseMask[1] == SM_SentinelZero, Subtarget, DAG,
+                            DL, 256);
     }
 
     if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128)
@@ -35460,12 +35448,9 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
       unsigned PermMask = 0;
       PermMask |= ((BaseMask[0] < 0 ? 0x8 : (BaseMask[0] & 1)) << 0);
       PermMask |= ((BaseMask[1] < 0 ? 0x8 : (BaseMask[1] & 1)) << 4);
-
-      Res = CanonicalizeShuffleInput(ShuffleVT, V1);
-      Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res,
-                        DAG.getUNDEF(ShuffleVT),
-                        DAG.getTargetConstant(PermMask, DL, MVT::i8));
-      return DAG.getBitcast(RootVT, Res);
+      return DAG.getNode(
+          X86ISD::VPERM2X128, DL, RootVT, CanonicalizeShuffleInput(RootVT, V1),
+          DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8));
     }
 
     if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
@@ -35481,14 +35466,12 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
         unsigned PermMask = 0;
         PermMask |= ((BaseMask[0] & 3) << 0);
         PermMask |= ((BaseMask[1] & 3) << 4);
-
         SDValue LHS = isInRange(BaseMask[0], 0, 2) ? V1 : V2;
         SDValue RHS = isInRange(BaseMask[1], 0, 2) ? V1 : V2;
-        Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT,
-                          CanonicalizeShuffleInput(ShuffleVT, LHS),
-                          CanonicalizeShuffleInput(ShuffleVT, RHS),
+        return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
+                          CanonicalizeShuffleInput(RootVT, LHS),
+                          CanonicalizeShuffleInput(RootVT, RHS),
                           DAG.getTargetConstant(PermMask, DL, MVT::i8));
-        return DAG.getBitcast(RootVT, Res);
       }
     }
   }
@@ -36008,12 +35991,16 @@ static SDValue combineX86ShuffleChainWithExtract(
   if (NumInputs == 0)
     return SDValue();
 
+  EVT RootVT = Root.getValueType();
+  unsigned RootSizeInBits = RootVT.getSizeInBits();
+  assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
+
   SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end());
   SmallVector<unsigned, 4> Offsets(NumInputs, 0);
 
   // Peek through subvectors.
   // TODO: Support inter-mixed EXTRACT_SUBVECTORs + BITCASTs?
-  unsigned WideSizeInBits = WideInputs[0].getValueSizeInBits();
+  unsigned WideSizeInBits = RootSizeInBits;
   for (unsigned i = 0; i != NumInputs; ++i) {
     SDValue &Src = WideInputs[i];
     unsigned &Offset = Offsets[i];
@@ -36036,8 +36023,6 @@ static SDValue combineX86ShuffleChainWithExtract(
   if (llvm::all_of(Offsets, [](unsigned Offset) { return Offset == 0; }))
     return SDValue();
 
-  EVT RootVT = Root.getValueType();
-  unsigned RootSizeInBits = RootVT.getSizeInBits();
   unsigned Scale = WideSizeInBits / RootSizeInBits;
   assert((WideSizeInBits % RootSizeInBits) == 0 &&
          "Unexpected subvector extraction");
@@ -37332,11 +37317,26 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
     return SDValue();
   }
   case X86ISD::VPERM2X128: {
+    // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)).
+    SDValue LHS = N->getOperand(0);
+    SDValue RHS = N->getOperand(1);
+    if (LHS.getOpcode() == ISD::BITCAST &&
+        (RHS.getOpcode() == ISD::BITCAST || RHS.isUndef())) {
+      EVT SrcVT = LHS.getOperand(0).getValueType();
+      if (RHS.isUndef() || SrcVT == RHS.getOperand(0).getValueType()) {
+        return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT,
+                                              DAG.getBitcast(SrcVT, LHS),
+                                              DAG.getBitcast(SrcVT, RHS),
+                                              N->getOperand(2)));
+      }
+    }
+
+    // Fold vperm2x128(op(),op()) -> op(vperm2x128(),vperm2x128()).
     if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL))
-        return Res;
+      return Res;
 
-    // Combine vperm2x128 subvector shuffle with an inner concat pattern.
-    // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc.
+    // Fold vperm2x128 subvector shuffle with an inner concat pattern.
+    // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc.  
     auto FindSubVector128 = [&](unsigned Idx) {
       if (Idx > 3)
         return SDValue();
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 071c638077b2..7cf555748c46 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -7287,16 +7287,12 @@ let ExeDomain = SSEPackedSingle in {
 let isCommutable = 1 in
 def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, u8imm:$src3),
-          "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          [(set VR256:$dst, (v4f64 (X86VPerm2x128 VR256:$src1, VR256:$src2,
-                              (i8 timm:$src3))))]>, VEX_4V, VEX_L,
-          Sched<[WriteFShuffle256]>;
+          "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,
+          VEX_4V, VEX_L, Sched<[WriteFShuffle256]>;
 def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f256mem:$src2, u8imm:$src3),
-          "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4f64 addr:$src2),
-                             (i8 timm:$src3)))]>, VEX_4V, VEX_L,
-          Sched<[WriteFShuffle256.Folded, WriteFShuffle256.ReadAfterFold]>;
+          "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,
+          VEX_4V, VEX_L, Sched<[WriteFShuffle256.Folded, WriteFShuffle256.ReadAfterFold]>;
 }
 
 // Immediate transform to help with commuting.
@@ -7304,23 +7300,27 @@ def Perm2XCommuteImm : SDNodeXForm<timm, [{
   return getI8Imm(N->getZExtValue() ^ 0x22, SDLoc(N));
 }]>;
 
+multiclass vperm2x128_lowering<string InstrStr, ValueType VT, PatFrag memop_frag> {
+  def : Pat<(VT (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 timm:$imm))),
+            (!cast<Instruction>(InstrStr#rr) VR256:$src1, VR256:$src2, timm:$imm)>;
+  def : Pat<(VT (X86VPerm2x128 VR256:$src1, (memop_frag addr:$src2), (i8 timm:$imm))),
+            (!cast<Instruction>(InstrStr#rm) VR256:$src1, addr:$src2, timm:$imm)>;
+  // Pattern with load in other operand.
+  def : Pat<(VT (X86VPerm2x128 (memop_frag addr:$src2), VR256:$src1, (i8 timm:$imm))),
+            (!cast<Instruction>(InstrStr#rm) VR256:$src1, addr:$src2,
+                                             (Perm2XCommuteImm timm:$imm))>;
+}
+
 let Predicates = [HasAVX] in {
-// Pattern with load in other operand.
-def : Pat<(v4f64 (X86VPerm2x128 (loadv4f64 addr:$src2),
-                                VR256:$src1, (i8 timm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm timm:$imm))>;
+  defm : vperm2x128_lowering<"VPERM2F128", v4f64, loadv4f64>;
+  defm : vperm2x128_lowering<"VPERM2F128", v8f32, loadv8f32>;
 }
 
 let Predicates = [HasAVX1Only] in {
-def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 timm:$imm))),
-          (VPERM2F128rr VR256:$src1, VR256:$src2, timm:$imm)>;
-def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1,
-                  (loadv4i64 addr:$src2), (i8 timm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, timm:$imm)>;
-// Pattern with load in other operand.
-def : Pat<(v4i64 (X86VPerm2x128 (loadv4i64 addr:$src2),
-                                VR256:$src1, (i8 timm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm timm:$imm))>;
+  defm : vperm2x128_lowering<"VPERM2F128", v4i64,  loadv4i64>;
+  defm : vperm2x128_lowering<"VPERM2F128", v8i32,  loadv8i32>;
+  defm : vperm2x128_lowering<"VPERM2F128", v16i16, loadv16i16>;
+  defm : vperm2x128_lowering<"VPERM2F128", v32i8,  loadv32i8>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7689,27 +7689,24 @@ defm VPERMPD : avx2_perm_imm<0x01, "vpermpd", loadv4f64, v4f64,
                              WriteFShuffle256, f256mem>, VEX_W;
 
 //===----------------------------------------------------------------------===//
-// VPERM2I128 - Permute Floating-Point Values in 128-bit chunks
+// VPERM2I128 - Permute Integer vector Values in 128-bit chunks
 //
 let isCommutable = 1 in
 def VPERM2I128rr : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, u8imm:$src3),
-          "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          [(set VR256:$dst, (v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2,
-                            (i8 timm:$src3))))]>, Sched<[WriteShuffle256]>,
-          VEX_4V, VEX_L;
+          "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,
+          Sched<[WriteShuffle256]>, VEX_4V, VEX_L;
 def VPERM2I128rm : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f256mem:$src2, u8imm:$src3),
-          "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4i64 addr:$src2),
-                             (i8 timm:$src3)))]>,
+          "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", []>,
           Sched<[WriteShuffle256.Folded, WriteShuffle256.ReadAfterFold]>, VEX_4V, VEX_L;
 
-let Predicates = [HasAVX2] in
-def : Pat<(v4i64 (X86VPerm2x128 (loadv4i64 addr:$src2),
-                                VR256:$src1, (i8 timm:$imm))),
-          (VPERM2I128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm timm:$imm))>;
-
+let Predicates = [HasAVX2] in {
+  defm : vperm2x128_lowering<"VPERM2I128", v4i64,  loadv4i64>;
+  defm : vperm2x128_lowering<"VPERM2I128", v8i32,  loadv8i32>;
+  defm : vperm2x128_lowering<"VPERM2I128", v16i16, loadv16i16>;
+  defm : vperm2x128_lowering<"VPERM2I128", v32i8,  loadv32i8>;
+}
 
 //===----------------------------------------------------------------------===//
 // VINSERTI128 - Insert packed integer values
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 3f290f2f087b..cd5c1863bb3c 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1549,7 +1549,6 @@ static void rewritePHIs(BasicBlock &BB) {
     for (BasicBlock *Pred : Preds) {
       if (CatchSwitchInst *CS =
               dyn_cast<CatchSwitchInst>(Pred->getTerminator())) {
-        (void)CS;
         // CleanupPad with a CatchSwitch predecessor: therefore this is an
         // unwind destination that needs to be handle specially.
         assert(CS->getUnwindDest() == &BB);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 5ba51d255109..0b4246feecee 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -830,6 +830,30 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
 
     break;
   }
+  case Intrinsic::umax:
+  case Intrinsic::umin: {
+    Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
+    Value *X, *Y;
+    if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_ZExt(m_Value(Y))) &&
+        (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
+      Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
+      return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
+    }
+    // If both operands of unsigned min/max are sign-extended, it is still ok
+    // to narrow the operation.
+    LLVM_FALLTHROUGH;
+  }
+  case Intrinsic::smax:
+  case Intrinsic::smin: {
+    Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
+    Value *X, *Y;
+    if (match(I0, m_SExt(m_Value(X))) && match(I1, m_SExt(m_Value(Y))) &&
+        (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
+      Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
+      return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
+    }
+    break;
+  }
   case Intrinsic::bswap: {
     Value *IIOperand = II->getArgOperand(0);
     Value *X = nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9373d40458f4..ea0d7673edf6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7718,9 +7718,15 @@ void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV,
 
   assert(BestVF.hasValue() && "Vectorization Factor is missing");
 
-  VPTransformState State{*BestVF, BestUF,      LI,
-                         DT,      ILV.Builder, ILV.VectorLoopValueMap,
-                         &ILV,    CallbackILV};
+  VPTransformState State{*BestVF,
+                         BestUF,
+                         OrigLoop,
+                         LI,
+                         DT,
+                         ILV.Builder,
+                         ILV.VectorLoopValueMap,
+                         &ILV,
+                         CallbackILV};
   State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
   State.TripCount = ILV.getOrCreateTripCount(nullptr);
   State.CanonicalIV = ILV.Induction;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7114b4d412fd..0b630197911a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3411,21 +3411,21 @@ bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
          });
 }
 
-static std::pair<unsigned, unsigned>
+static std::pair<InstructionCost, InstructionCost>
 getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
                    TargetTransformInfo *TTI, TargetLibraryInfo *TLI) {
   Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
 
   // Calculate the cost of the scalar and vector calls.
   IntrinsicCostAttributes CostAttrs(ID, *CI, VecTy->getElementCount());
-  int IntrinsicCost =
+  auto IntrinsicCost =
     TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
 
   auto Shape = VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
                                      VecTy->getNumElements())),
                             false /*HasGlobalPred*/);
   Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
-  int LibCost = IntrinsicCost;
+  auto LibCost = IntrinsicCost;
   if (!CI->isNoBuiltin() && VecFunc) {
     // Calculate the cost of the vector library call.
     SmallVector<Type *, 4> VecTys;
@@ -5994,7 +5994,7 @@ bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
   InstructionCost Cost = R.getTreeCost();
 
   LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for VF =" << VF << "\n");
-  if (Cost.isValid() && Cost < -SLPCostThreshold) {
+  if (Cost < -SLPCostThreshold) {
     LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost = " << Cost << "\n");
 
     using namespace ore;
@@ -6295,7 +6295,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
 
       MinCost = std::min(MinCost, Cost);
 
-      if (Cost.isValid() && Cost < -SLPCostThreshold) {
+      if (Cost < -SLPCostThreshold) {
         LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
         R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
                                                     cast<Instruction>(Ops[0]))
@@ -7007,11 +7007,12 @@ class HorizontalReduction {
 
 private:
   /// Calculate the cost of a reduction.
-  int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
-                       unsigned ReduxWidth) {
+  InstructionCost getReductionCost(TargetTransformInfo *TTI,
+                                   Value *FirstReducedVal,
+                                   unsigned ReduxWidth) {
     Type *ScalarTy = FirstReducedVal->getType();
     FixedVectorType *VectorTy = FixedVectorType::get(ScalarTy, ReduxWidth);
-    int VectorCost, ScalarCost;
+    InstructionCost VectorCost, ScalarCost;
     switch (RdxKind) {
     case RecurKind::Add:
     case RecurKind::Mul:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 120562b6db5f..b26399e0ae58 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -216,6 +216,27 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() {
   return It;
 }
 
+Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
+  if (!Def->getDef() && OrigLoop->isLoopInvariant(Def->getLiveInIRValue()))
+    return Def->getLiveInIRValue();
+
+  if (hasScalarValue(Def, Instance))
+    return Data.PerPartScalars[Def][Instance.Part][Instance.Lane];
+
+  if (hasVectorValue(Def, Instance.Part)) {
+    assert(Data.PerPartOutput.count(Def));
+    auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
+    if (!VecPart->getType()->isVectorTy()) {
+      assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
+      return VecPart;
+    }
+    // TODO: Cache created scalar values.
+    return Builder.CreateExtractElement(VecPart,
+                                        Builder.getInt32(Instance.Lane));
+  }
+  return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance);
+}
+
 BasicBlock *
 VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
   // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8d907dbc857d..2cce127cd4ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -246,12 +246,12 @@ struct VPCallback {
 /// VPTransformState holds information passed down when "executing" a VPlan,
 /// needed for generating the output IR.
 struct VPTransformState {
-  VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI,
+  VPTransformState(ElementCount VF, unsigned UF, Loop *OrigLoop, LoopInfo *LI,
                    DominatorTree *DT, IRBuilder<> &Builder,
                    VectorizerValueMap &ValueMap, InnerLoopVectorizer *ILV,
                    VPCallback &Callback)
-      : VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder),
-        ValueMap(ValueMap), ILV(ILV), Callback(Callback) {}
+      : VF(VF), UF(UF), Instance(), OrigLoop(OrigLoop), LI(LI), DT(DT),
+        Builder(Builder), ValueMap(ValueMap), ILV(ILV), Callback(Callback) {}
 
   /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
   ElementCount VF;
@@ -269,6 +269,9 @@ struct VPTransformState {
     typedef SmallVector<Value *, 2> PerPartValuesTy;
 
     DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
+
+    using ScalarsPerPartValuesTy = SmallVector<SmallVector<Value *, 4>, 2>;
+    DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
   } Data;
 
   /// Get the generated Value for a given VPValue and a given Part. Note that
@@ -285,24 +288,21 @@ struct VPTransformState {
   }
 
   /// Get the generated Value for a given VPValue and given Part and Lane.
-  Value *get(VPValue *Def, const VPIteration &Instance) {
-    // If the Def is managed directly by VPTransformState, extract the lane from
-    // the relevant part. Note that currently only VPInstructions and external
-    // defs are managed by VPTransformState. Other Defs are still created by ILV
-    // and managed in its ValueMap. For those this method currently just
-    // delegates the call to ILV below.
-    if (Data.PerPartOutput.count(Def)) {
-      auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
-      if (!VecPart->getType()->isVectorTy()) {
-        assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
-        return VecPart;
-      }
-      // TODO: Cache created scalar values.
-      return Builder.CreateExtractElement(VecPart,
-                                          Builder.getInt32(Instance.Lane));
-    }
+  Value *get(VPValue *Def, const VPIteration &Instance);
 
-    return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance);
+  bool hasVectorValue(VPValue *Def, unsigned Part) {
+    auto I = Data.PerPartOutput.find(Def);
+    return I != Data.PerPartOutput.end() && Part < I->second.size() &&
+           I->second[Part];
+  }
+
+  bool hasScalarValue(VPValue *Def, VPIteration Instance) {
+    auto I = Data.PerPartScalars.find(Def);
+    if (I == Data.PerPartScalars.end())
+      return false;
+    return Instance.Part < I->second.size() &&
+           Instance.Lane < I->second[Instance.Part].size() &&
+           I->second[Instance.Part][Instance.Lane];
   }
 
   /// Set the generated Value for a given VPValue and a given Part.
@@ -315,6 +315,17 @@ struct VPTransformState {
   }
   void set(VPValue *Def, Value *IRDef, Value *V, unsigned Part);
 
+  void set(VPValue *Def, Value *V, const VPIteration &Instance) {
+    auto Iter = Data.PerPartScalars.insert({Def, {}});
+    auto &PerPartVec = Iter.first->second;
+    while (PerPartVec.size() <= Instance.Part)
+      PerPartVec.emplace_back();
+    auto &Scalars = PerPartVec[Instance.Part];
+    while (Scalars.size() <= Instance.Lane)
+      Scalars.push_back(nullptr);
+    Scalars[Instance.Lane] = V;
+  }
+
   /// Hold state information used when constructing the CFG of the output IR,
   /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
   struct CFGState {
@@ -340,6 +351,9 @@ struct VPTransformState {
     CFGState() = default;
   } CFG;
 
+  /// Hold a pointer to the original loop.
+  Loop *OrigLoop;
+
   /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
   LoopInfo *LI;
 
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-addr.ll b/llvm/test/CodeGen/AArch64/misched-fusion-addr.ll
index 694c8a0d0765..489cc849b908 100644
--- a/llvm/test/CodeGen/AArch64/misched-fusion-addr.ll
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-addr.ll
@@ -1,4 +1,5 @@
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mattr=fuse-address | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a65    | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m3     | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m4     | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m5     | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
index 95a419bd7398..ef01326f376c 100644
--- a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
@@ -2,8 +2,14 @@
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=generic -mattr=+crypto | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a53 | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a57 | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a65 | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a72 | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a73 | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a76 | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a77 | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a78 | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a78c| FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-x1  | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m3  | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m4  | FileCheck %s
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m5  | FileCheck %s
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-lit.ll b/llvm/test/CodeGen/AArch64/misched-fusion-lit.ll
index 74180d92089b..be67e9f852e1 100644
--- a/llvm/test/CodeGen/AArch64/misched-fusion-lit.ll
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-lit.ll
@@ -1,6 +1,8 @@
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mattr=-fuse-literals | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKDONT
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mattr=+fuse-literals | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a57      | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a65      | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a72      | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m3       | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m4       | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
 ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m5       | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
index a45c759db58e..ad909c00fa37 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll
@@ -593,7 +593,7 @@ define <vscale x 2 x i64> @mul_i64_range(<vscale x 2 x i64> %a) {
 
 define <vscale x 16 x i8> @asr_i8(<vscale x 16 x i8> %a){
 ; CHECK-LABEL: @asr_i8
-; CHECK-DAG: asr z0.b, z0.b, #8
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
   %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -603,7 +603,7 @@ define <vscale x 16 x i8> @asr_i8(<vscale x 16 x i8> %a){
 
 define <vscale x 8 x i16> @asr_i16(<vscale x 8 x i16> %a){
 ; CHECK-LABEL: @asr_i16
-; CHECK-DAG: asr z0.h, z0.h, #16
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 8 x i16> undef, i16 16, i32 0
   %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -613,7 +613,7 @@ define <vscale x 8 x i16> @asr_i16(<vscale x 8 x i16> %a){
 
 define <vscale x 4 x i32> @asr_i32(<vscale x 4 x i32> %a){
 ; CHECK-LABEL: @asr_i32
-; CHECK-DAG: asr z0.s, z0.s, #32
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 4 x i32> undef, i32 32, i32 0
   %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -623,7 +623,7 @@ define <vscale x 4 x i32> @asr_i32(<vscale x 4 x i32> %a){
 
 define <vscale x 2 x i64> @asr_i64(<vscale x 2 x i64> %a){
 ; CHECK-LABEL: @asr_i64
-; CHECK-DAG: asr z0.d, z0.d, #64
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 2 x i64> undef, i64 64, i32 0
   %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -677,7 +677,7 @@ define <vscale x 2 x i64> @lsl_i64(<vscale x 2 x i64> %a){
 
 define <vscale x 16 x i8> @lsr_i8(<vscale x 16 x i8> %a){
 ; CHECK-LABEL: @lsr_i8
-; CHECK-DAG: lsr z0.b, z0.b, #8
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 16 x i8> undef, i8 8, i32 0
   %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -687,7 +687,7 @@ define <vscale x 16 x i8> @lsr_i8(<vscale x 16 x i8> %a){
 
 define <vscale x 8 x i16> @lsr_i16(<vscale x 8 x i16> %a){
 ; CHECK-LABEL: @lsr_i16
-; CHECK-DAG: lsr z0.h, z0.h, #16
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 8 x i16> undef, i16 16, i32 0
   %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -697,7 +697,7 @@ define <vscale x 8 x i16> @lsr_i16(<vscale x 8 x i16> %a){
 
 define <vscale x 4 x i32> @lsr_i32(<vscale x 4 x i32> %a){
 ; CHECK-LABEL: @lsr_i32
-; CHECK-DAG: lsr z0.s, z0.s, #32
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 4 x i32> undef, i32 32, i32 0
   %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -707,7 +707,7 @@ define <vscale x 4 x i32> @lsr_i32(<vscale x 4 x i32> %a){
 
 define <vscale x 2 x i64> @lsr_i64(<vscale x 2 x i64> %a){
 ; CHECK-LABEL: @lsr_i64
-; CHECK-DAG: lsr z0.d, z0.d, #64
+; CHECK: // %bb.0:
 ; CHECK-NEXT: ret
   %elt = insertelement <vscale x 2 x i64> undef, i64 64, i32 0
   %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index b36b4430f157..a1e216096b1f 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -5,22 +5,84 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+f %s -o - | FileCheck --check-prefix=RV32F %s
 ; RUN: llc -mtriple=riscv32 -mattr=+d %s -o - | FileCheck --check-prefix=RV32D %s
 ; RUN: llc -mtriple=riscv32 -mattr=+c %s -o - | FileCheck --check-prefix=RV32C %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-b %s -o - | FileCheck --check-prefix=RV32B %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v,+experimental-zvamo,+experimental-zvlsseg %s -o - | FileCheck --check-prefix=RV32V %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh %s -o - | FileCheck --check-prefix=RV32ZFH %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zba %s -o - | FileCheck --check-prefix=RV32ZBA %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbb %s -o - | FileCheck --check-prefix=RV32ZBB %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbc %s -o - | FileCheck --check-prefix=RV32ZBC %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbe %s -o - | FileCheck --check-prefix=RV32ZBE %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbf %s -o - | FileCheck --check-prefix=RV32ZBF %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbm %s -o - | FileCheck --check-prefix=RV32ZBM %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbp %s -o - | FileCheck --check-prefix=RV32ZBP %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbproposedc %s -o - | FileCheck --check-prefix=RV32ZBPROPOSEDC %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbr %s -o - | FileCheck --check-prefix=RV32ZBR %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbs %s -o - | FileCheck --check-prefix=RV32ZBS %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbt %s -o - | FileCheck --check-prefix=RV32ZBT %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zbb,+experimental-zfh,+experimental-zvamo,+experimental-v,+f,+experimental-zvlsseg %s -o - | FileCheck --check-prefix=RV32COMBINED %s
 ; RUN: llc -mtriple=riscv64 -mattr=+m %s -o - | FileCheck --check-prefix=RV64M %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a %s -o - | FileCheck --check-prefix=RV64A %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f %s -o - | FileCheck --check-prefix=RV64F %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d %s -o - | FileCheck --check-prefix=RV64D %s
 ; RUN: llc -mtriple=riscv64 -mattr=+c %s -o - | FileCheck --check-prefix=RV64C %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-b %s -o - | FileCheck --check-prefix=RV64B %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-v,+experimental-zvamo,+experimental-zvlsseg %s -o - | FileCheck --check-prefix=RV64V %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh %s -o - | FileCheck --check-prefix=RV64ZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zba %s -o - | FileCheck --check-prefix=RV64ZBA %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbb %s -o - | FileCheck --check-prefix=RV64ZBB %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbc %s -o - | FileCheck --check-prefix=RV64ZBC %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbe %s -o - | FileCheck --check-prefix=RV64ZBE %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbf %s -o - | FileCheck --check-prefix=RV64ZBF %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbm %s -o - | FileCheck --check-prefix=RV64ZBM %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbp %s -o - | FileCheck --check-prefix=RV64ZBP %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbproposedc %s -o - | FileCheck --check-prefix=RV64ZBPROPOSEDC %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbr %s -o - | FileCheck --check-prefix=RV64ZBR %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbs %s -o - | FileCheck --check-prefix=RV64ZBS %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbt %s -o - | FileCheck --check-prefix=RV64ZBT %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zbb,+experimental-zfh,+experimental-zvamo,+experimental-v,+f,+experimental-zvlsseg %s -o - | FileCheck --check-prefix=RV64COMBINED %s
 
 ; RV32M: .attribute 5, "rv32i2p0_m2p0"
 ; RV32A: .attribute 5, "rv32i2p0_a2p0"
 ; RV32F: .attribute 5, "rv32i2p0_f2p0"
 ; RV32D: .attribute 5, "rv32i2p0_f2p0_d2p0"
 ; RV32C: .attribute 5, "rv32i2p0_c2p0"
+; RV32B: .attribute 5, "rv32i2p0_b0p93_zba0p93_zbb0p93_zbc0p93_zbe0p93_zbf0p93_zbm0p93_zbp0p93_zbr0p93_zbs0p93_zbt0p93"
+; RV32V: .attribute 5, "rv32i2p0_v0p9_zvamo0p9_zvlsseg0p9"
+; RV32ZFH: .attribute 5, "rv32i2p0_f2p0_zfh0p1"
+; RV32ZBA: .attribute 5, "rv32i2p0_zba0p93"
+; RV32ZBB: .attribute 5, "rv32i2p0_zbb0p93"
+; RV32ZBC: .attribute 5, "rv32i2p0_zbc0p93"
+; RV32ZBE: .attribute 5, "rv32i2p0_zbe0p93"
+; RV32ZBF: .attribute 5, "rv32i2p0_zbf0p93"
+; RV32ZBM: .attribute 5, "rv32i2p0_zbm0p93"
+; RV32ZBP: .attribute 5, "rv32i2p0_zbp0p93"
+; RV32ZBPROPOSEDC: .attribute 5, "rv32i2p0_zbproposedc0p93"
+; RV32ZBR: .attribute 5, "rv32i2p0_zbr0p93"
+; RV32ZBS: .attribute 5, "rv32i2p0_zbs0p93"
+; RV32ZBT: .attribute 5, "rv32i2p0_zbt0p93"
+; RV32COMBINED: .attribute 5, "rv32i2p0_f2p0_v0p9_zfh0p1_zbb0p93_zvamo0p9_zvlsseg0p9"
+
 ; RV64M: .attribute 5, "rv64i2p0_m2p0"
 ; RV64A: .attribute 5, "rv64i2p0_a2p0"
 ; RV64F: .attribute 5, "rv64i2p0_f2p0"
 ; RV64D: .attribute 5, "rv64i2p0_f2p0_d2p0"
 ; RV64C: .attribute 5, "rv64i2p0_c2p0"
+; RV64B: .attribute 5, "rv64i2p0_b0p93_zba0p93_zbb0p93_zbc0p93_zbe0p93_zbf0p93_zbm0p93_zbp0p93_zbr0p93_zbs0p93_zbt0p93"
+; RV64ZFH: .attribute 5, "rv64i2p0_f2p0_zfh0p1"
+; RV64ZBA: .attribute 5, "rv64i2p0_zba0p93"
+; RV64ZBB: .attribute 5, "rv64i2p0_zbb0p93"
+; RV64ZBC: .attribute 5, "rv64i2p0_zbc0p93"
+; RV64ZBE: .attribute 5, "rv64i2p0_zbe0p93"
+; RV64ZBF: .attribute 5, "rv64i2p0_zbf0p93"
+; RV64ZBM: .attribute 5, "rv64i2p0_zbm0p93"
+; RV64ZBP: .attribute 5, "rv64i2p0_zbp0p93"
+; RV64ZBPROPOSEDC: .attribute 5, "rv64i2p0_zbproposedc0p93"
+; RV64ZBR: .attribute 5, "rv64i2p0_zbr0p93"
+; RV64ZBS: .attribute 5, "rv64i2p0_zbs0p93"
+; RV64ZBT: .attribute 5, "rv64i2p0_zbt0p93"
+; RV64V: .attribute 5, "rv64i2p0_v0p9_zvamo0p9_zvlsseg0p9"
+; RV64COMBINED: .attribute 5, "rv64i2p0_f2p0_v0p9_zfh0p1_zbb0p93_zvamo0p9_zvlsseg0p9"
+
 
 define i32 @addi(i32 %a) {
   %1 = add i32 %a, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv32.ll
index 7de6430d3891..239151274c4e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv32.ll
@@ -26,9 +26,15 @@ define <vscale x 1 x i8> @vdiv_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b) {
 define <vscale x 1 x i8> @vdiv_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -85,9 +91,15 @@ define <vscale x 2 x i8> @vdiv_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b) {
 define <vscale x 2 x i8> @vdiv_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -120,9 +132,15 @@ define <vscale x 4 x i8> @vdiv_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b) {
 define <vscale x 4 x i8> @vdiv_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -155,9 +173,15 @@ define <vscale x 8 x i8> @vdiv_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b) {
 define <vscale x 8 x i8> @vdiv_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -190,9 +214,15 @@ define <vscale x 16 x i8> @vdiv_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %b
 define <vscale x 16 x i8> @vdiv_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vrsub.vi v28, v8, 0
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vsrl.vi v28, v26, 7
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -225,9 +255,15 @@ define <vscale x 32 x i8> @vdiv_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %b
 define <vscale x 32 x i8> @vdiv_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vrsub.vi v8, v8, 0
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vsra.vi v28, v28, 2
+; CHECK-NEXT:    vsrl.vi v8, v28, 7
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -260,9 +296,15 @@ define <vscale x 64 x i8> @vdiv_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %b
 define <vscale x 64 x i8> @vdiv_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vrsub.vi v8, v8, 0
+; CHECK-NEXT:    vadd.vv v8, v16, v8
+; CHECK-NEXT:    vsra.vi v8, v8, 2
+; CHECK-NEXT:    vsrl.vi v16, v8, 7
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -295,9 +337,15 @@ define <vscale x 1 x i16> @vdiv_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext %
 define <vscale x 1 x i16> @vdiv_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -330,9 +378,15 @@ define <vscale x 2 x i16> @vdiv_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext %
 define <vscale x 2 x i16> @vdiv_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -365,9 +419,15 @@ define <vscale x 4 x i16> @vdiv_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext %
 define <vscale x 4 x i16> @vdiv_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -400,9 +460,15 @@ define <vscale x 8 x i16> @vdiv_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext %
 define <vscale x 8 x i16> @vdiv_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v26, v26, 1
+; CHECK-NEXT:    vsrl.vi v28, v26, 15
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -435,9 +501,15 @@ define <vscale x 16 x i16> @vdiv_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signex
 define <vscale x 16 x i16> @vdiv_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v28, v28, 1
+; CHECK-NEXT:    vsrl.vi v8, v28, 15
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -470,9 +542,15 @@ define <vscale x 32 x i16> @vdiv_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signex
 define <vscale x 32 x i16> @vdiv_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v8, v8, a0
+; CHECK-NEXT:    vadd.vi v8, v8, 0
+; CHECK-NEXT:    vsra.vi v8, v8, 1
+; CHECK-NEXT:    vsrl.vi v16, v8, 15
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -505,9 +583,15 @@ define <vscale x 1 x i32> @vdiv_vx_nxv1i32(<vscale x 1 x i32> %va, i32 %b) {
 define <vscale x 1 x i32> @vdiv_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vsub.vv v25, v25, v8
+; CHECK-NEXT:    vsra.vi v26, v25, 2
+; CHECK-NEXT:    vsrl.vi v25, v25, 31
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -540,9 +624,15 @@ define <vscale x 2 x i32> @vdiv_vx_nxv2i32(<vscale x 2 x i32> %va, i32 %b) {
 define <vscale x 2 x i32> @vdiv_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vsub.vv v25, v25, v8
+; CHECK-NEXT:    vsra.vi v26, v25, 2
+; CHECK-NEXT:    vsrl.vi v25, v25, 31
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -575,9 +665,15 @@ define <vscale x 4 x i32> @vdiv_vx_nxv4i32(<vscale x 4 x i32> %va, i32 %b) {
 define <vscale x 4 x i32> @vdiv_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vsub.vv v26, v26, v8
+; CHECK-NEXT:    vsra.vi v28, v26, 2
+; CHECK-NEXT:    vsrl.vi v26, v26, 31
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -610,9 +706,15 @@ define <vscale x 8 x i32> @vdiv_vx_nxv8i32(<vscale x 8 x i32> %va, i32 %b) {
 define <vscale x 8 x i32> @vdiv_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vsub.vv v28, v28, v8
+; CHECK-NEXT:    vsra.vi v8, v28, 2
+; CHECK-NEXT:    vsrl.vi v28, v28, 31
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -645,9 +747,15 @@ define <vscale x 16 x i32> @vdiv_vx_nxv16i32(<vscale x 16 x i32> %va, i32 %b) {
 define <vscale x 16 x i32> @vdiv_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vsub.vv v8, v16, v8
+; CHECK-NEXT:    vsra.vi v16, v8, 2
+; CHECK-NEXT:    vsrl.vi v8, v8, 31
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -687,9 +795,25 @@ define <vscale x 1 x i64> @vdiv_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vdiv_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v25, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v25, v25, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v26, a1
+; CHECK-NEXT:    vsll.vx v26, v26, a0
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vor.vv v25, v26, v25
+; CHECK-NEXT:    vmulh.vv v25, v8, v25
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v26, v25, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v25, v25, a0
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -729,9 +853,25 @@ define <vscale x 2 x i64> @vdiv_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vdiv_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v26, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v26, v26, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v28, a1
+; CHECK-NEXT:    vsll.vx v28, v28, a0
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vor.vv v26, v28, v26
+; CHECK-NEXT:    vmulh.vv v26, v8, v26
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v28, v26, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -771,9 +911,25 @@ define <vscale x 4 x i64> @vdiv_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vdiv_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v28, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v28, v28, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v12, a1
+; CHECK-NEXT:    vsll.vx v12, v12, a0
+; CHECK-NEXT:    vsrl.vx v12, v12, a0
+; CHECK-NEXT:    vor.vv v28, v12, v28
+; CHECK-NEXT:    vmulh.vv v28, v8, v28
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v8, v28, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -813,9 +969,25 @@ define <vscale x 8 x i64> @vdiv_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vdiv_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v16, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v16, v16, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v24, a1
+; CHECK-NEXT:    vsll.vx v24, v24, a0
+; CHECK-NEXT:    vsrl.vx v24, v24, a0
+; CHECK-NEXT:    vor.vv v16, v24, v16
+; CHECK-NEXT:    vmulh.vv v8, v8, v16
+; CHECK-NEXT:    vadd.vi v8, v8, 0
+; CHECK-NEXT:    vsra.vi v16, v8, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv64.ll
index 51268e0edd4e..991cccf72cdd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdiv-sdnode-rv64.ll
@@ -26,9 +26,15 @@ define <vscale x 1 x i8> @vdiv_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b) {
 define <vscale x 1 x i8> @vdiv_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -61,9 +67,15 @@ define <vscale x 2 x i8> @vdiv_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b) {
 define <vscale x 2 x i8> @vdiv_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -96,9 +108,15 @@ define <vscale x 4 x i8> @vdiv_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b) {
 define <vscale x 4 x i8> @vdiv_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -131,9 +149,15 @@ define <vscale x 8 x i8> @vdiv_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b) {
 define <vscale x 8 x i8> @vdiv_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -166,9 +190,15 @@ define <vscale x 16 x i8> @vdiv_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %b
 define <vscale x 16 x i8> @vdiv_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vrsub.vi v28, v8, 0
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vsrl.vi v28, v26, 7
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -201,9 +231,15 @@ define <vscale x 32 x i8> @vdiv_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %b
 define <vscale x 32 x i8> @vdiv_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vrsub.vi v8, v8, 0
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vsra.vi v28, v28, 2
+; CHECK-NEXT:    vsrl.vi v8, v28, 7
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -236,9 +272,15 @@ define <vscale x 64 x i8> @vdiv_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %b
 define <vscale x 64 x i8> @vdiv_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vrsub.vi v8, v8, 0
+; CHECK-NEXT:    vadd.vv v8, v16, v8
+; CHECK-NEXT:    vsra.vi v8, v8, 2
+; CHECK-NEXT:    vsrl.vi v16, v8, 7
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -271,9 +313,15 @@ define <vscale x 1 x i16> @vdiv_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext %
 define <vscale x 1 x i16> @vdiv_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -306,9 +354,15 @@ define <vscale x 2 x i16> @vdiv_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext %
 define <vscale x 2 x i16> @vdiv_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -341,9 +395,15 @@ define <vscale x 4 x i16> @vdiv_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext %
 define <vscale x 4 x i16> @vdiv_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -376,9 +436,15 @@ define <vscale x 8 x i16> @vdiv_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext %
 define <vscale x 8 x i16> @vdiv_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v26, v26, 1
+; CHECK-NEXT:    vsrl.vi v28, v26, 15
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -411,9 +477,15 @@ define <vscale x 16 x i16> @vdiv_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signex
 define <vscale x 16 x i16> @vdiv_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v28, v28, 1
+; CHECK-NEXT:    vsrl.vi v8, v28, 15
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -446,9 +518,15 @@ define <vscale x 32 x i16> @vdiv_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signex
 define <vscale x 32 x i16> @vdiv_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v8, v8, a0
+; CHECK-NEXT:    vadd.vi v8, v8, 0
+; CHECK-NEXT:    vsra.vi v8, v8, 1
+; CHECK-NEXT:    vsrl.vi v16, v8, 15
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -481,9 +559,16 @@ define <vscale x 1 x i32> @vdiv_vx_nxv1i32(<vscale x 1 x i32> %va, i32 signext %
 define <vscale x 1 x i32> @vdiv_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 31
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -516,9 +601,16 @@ define <vscale x 2 x i32> @vdiv_vx_nxv2i32(<vscale x 2 x i32> %va, i32 signext %
 define <vscale x 2 x i32> @vdiv_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 31
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v25, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -551,9 +643,16 @@ define <vscale x 4 x i32> @vdiv_vx_nxv4i32(<vscale x 4 x i32> %va, i32 signext %
 define <vscale x 4 x i32> @vdiv_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vrsub.vi v28, v8, 0
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vsrl.vi v28, v26, 31
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -586,9 +685,16 @@ define <vscale x 8 x i32> @vdiv_vx_nxv8i32(<vscale x 8 x i32> %va, i32 signext %
 define <vscale x 8 x i32> @vdiv_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vrsub.vi v8, v8, 0
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vsra.vi v28, v28, 2
+; CHECK-NEXT:    vsrl.vi v8, v28, 31
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -621,9 +727,16 @@ define <vscale x 16 x i32> @vdiv_vx_nxv16i32(<vscale x 16 x i32> %va, i32 signex
 define <vscale x 16 x i32> @vdiv_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vrsub.vi v8, v8, 0
+; CHECK-NEXT:    vadd.vv v8, v16, v8
+; CHECK-NEXT:    vsra.vi v8, v8, 2
+; CHECK-NEXT:    vsrl.vi v16, v8, 31
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -656,9 +769,22 @@ define <vscale x 1 x i64> @vdiv_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vdiv_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v26, v25, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v25, v25, a0
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v8, v26, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -691,9 +817,22 @@ define <vscale x 2 x i64> @vdiv_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vdiv_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v28, v26, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v8, v28, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -726,9 +865,22 @@ define <vscale x 4 x i64> @vdiv_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vdiv_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v8, v28, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -761,9 +913,22 @@ define <vscale x 8 x i64> @vdiv_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vdiv_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vdiv_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vdivu.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v8, v8, a0
+; CHECK-NEXT:    vadd.vi v8, v8, 0
+; CHECK-NEXT:    vsra.vi v16, v8, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    vand.vi v8, v8, -1
+; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv32.ll
index bc817329a321..27b27cd64bae 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv32.ll
@@ -26,9 +26,13 @@ define <vscale x 1 x i8> @vdivu_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b)
 define <vscale x 1 x i8> @vdivu_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -85,9 +89,13 @@ define <vscale x 2 x i8> @vdivu_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b)
 define <vscale x 2 x i8> @vdivu_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -120,9 +128,13 @@ define <vscale x 4 x i8> @vdivu_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b)
 define <vscale x 4 x i8> @vdivu_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -155,9 +167,13 @@ define <vscale x 8 x i8> @vdivu_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b)
 define <vscale x 8 x i8> @vdivu_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -190,9 +206,13 @@ define <vscale x 16 x i8> @vdivu_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %
 define <vscale x 16 x i8> @vdivu_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 5
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -225,9 +245,13 @@ define <vscale x 32 x i8> @vdivu_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %
 define <vscale x 32 x i8> @vdivu_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 5
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -260,9 +284,13 @@ define <vscale x 64 x i8> @vdivu_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %
 define <vscale x 64 x i8> @vdivu_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 5
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -295,9 +323,14 @@ define <vscale x 1 x i16> @vdivu_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext
 define <vscale x 1 x i16> @vdivu_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -330,9 +363,14 @@ define <vscale x 2 x i16> @vdivu_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext
 define <vscale x 2 x i16> @vdivu_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -365,9 +403,14 @@ define <vscale x 4 x i16> @vdivu_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext
 define <vscale x 4 x i16> @vdivu_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -400,9 +443,14 @@ define <vscale x 8 x i16> @vdivu_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext
 define <vscale x 8 x i16> @vdivu_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 13
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -435,9 +483,14 @@ define <vscale x 16 x i16> @vdivu_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signe
 define <vscale x 16 x i16> @vdivu_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 13
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -470,9 +523,14 @@ define <vscale x 32 x i16> @vdivu_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signe
 define <vscale x 32 x i16> @vdivu_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 13
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -505,9 +563,14 @@ define <vscale x 1 x i32> @vdivu_vx_nxv1i32(<vscale x 1 x i32> %va, i32 %b) {
 define <vscale x 1 x i32> @vdivu_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 29
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -540,9 +603,14 @@ define <vscale x 2 x i32> @vdivu_vx_nxv2i32(<vscale x 2 x i32> %va, i32 %b) {
 define <vscale x 2 x i32> @vdivu_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 29
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -575,9 +643,14 @@ define <vscale x 4 x i32> @vdivu_vx_nxv4i32(<vscale x 4 x i32> %va, i32 %b) {
 define <vscale x 4 x i32> @vdivu_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; CHECK-NEXT:    vmv.v.i v26, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 29
+; CHECK-NEXT:    vmseq.vi v0, v26, 1
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -610,9 +683,14 @@ define <vscale x 8 x i32> @vdivu_vx_nxv8i32(<vscale x 8 x i32> %va, i32 %b) {
 define <vscale x 8 x i32> @vdivu_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v12, v8, a0
+; CHECK-NEXT:    vsrl.vi v12, v12, 29
+; CHECK-NEXT:    vmseq.vi v0, v28, 1
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -645,9 +723,14 @@ define <vscale x 16 x i32> @vdivu_vx_nxv16i32(<vscale x 16 x i32> %va, i32 %b) {
 define <vscale x 16 x i32> @vdivu_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
+; CHECK-NEXT:    vmv.v.i v16, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v24, v8, a0
+; CHECK-NEXT:    vsrl.vi v24, v24, 29
+; CHECK-NEXT:    vmseq.vi v0, v16, 1
+; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -687,9 +770,21 @@ define <vscale x 1 x i64> @vdivu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vdivu_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v26, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v26, v26, a0
+; CHECK-NEXT:    vmv.v.i v27, 1
+; CHECK-NEXT:    vsll.vx v27, v27, a0
+; CHECK-NEXT:    vsrl.vx v27, v27, a0
+; CHECK-NEXT:    vor.vv v26, v27, v26
+; CHECK-NEXT:    vmulhu.vv v26, v8, v26
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -729,9 +824,21 @@ define <vscale x 2 x i64> @vdivu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vdivu_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; CHECK-NEXT:    vmv.v.i v26, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v28, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v28, v28, a0
+; CHECK-NEXT:    vmv.v.i v30, 1
+; CHECK-NEXT:    vsll.vx v30, v30, a0
+; CHECK-NEXT:    vsrl.vx v30, v30, a0
+; CHECK-NEXT:    vor.vv v28, v30, v28
+; CHECK-NEXT:    vmulhu.vv v28, v8, v28
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vmseq.vi v0, v26, 1
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -771,9 +878,21 @@ define <vscale x 4 x i64> @vdivu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vdivu_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v12, v12, a0
+; CHECK-NEXT:    vmv.v.i v16, 1
+; CHECK-NEXT:    vsll.vx v16, v16, a0
+; CHECK-NEXT:    vsrl.vx v16, v16, a0
+; CHECK-NEXT:    vor.vv v12, v16, v12
+; CHECK-NEXT:    vmulhu.vv v12, v8, v12
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v12, v12, a0
+; CHECK-NEXT:    vmseq.vi v0, v28, 1
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -813,9 +932,21 @@ define <vscale x 8 x i64> @vdivu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vdivu_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vmv.v.i v16, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v24, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v24, v24, a0
+; CHECK-NEXT:    vmv.v.i v0, 1
+; CHECK-NEXT:    vsll.vx v0, v0, a0
+; CHECK-NEXT:    vsrl.vx v0, v0, a0
+; CHECK-NEXT:    vor.vv v24, v0, v24
+; CHECK-NEXT:    vmulhu.vv v24, v8, v24
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v24, v24, a0
+; CHECK-NEXT:    vmseq.vi v0, v16, 1
+; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv64.ll
index 5f9b2d220ba2..70cd4fba1eb7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vdivu-sdnode-rv64.ll
@@ -26,9 +26,13 @@ define <vscale x 1 x i8> @vdivu_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b)
 define <vscale x 1 x i8> @vdivu_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -61,9 +65,13 @@ define <vscale x 2 x i8> @vdivu_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b)
 define <vscale x 2 x i8> @vdivu_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -96,9 +104,13 @@ define <vscale x 4 x i8> @vdivu_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b)
 define <vscale x 4 x i8> @vdivu_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -131,9 +143,13 @@ define <vscale x 8 x i8> @vdivu_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b)
 define <vscale x 8 x i8> @vdivu_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -166,9 +182,13 @@ define <vscale x 16 x i8> @vdivu_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %
 define <vscale x 16 x i8> @vdivu_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 5
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -201,9 +221,13 @@ define <vscale x 32 x i8> @vdivu_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %
 define <vscale x 32 x i8> @vdivu_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 5
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -236,9 +260,13 @@ define <vscale x 64 x i8> @vdivu_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %
 define <vscale x 64 x i8> @vdivu_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 5
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -271,9 +299,14 @@ define <vscale x 1 x i16> @vdivu_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext
 define <vscale x 1 x i16> @vdivu_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -306,9 +339,14 @@ define <vscale x 2 x i16> @vdivu_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext
 define <vscale x 2 x i16> @vdivu_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -341,9 +379,14 @@ define <vscale x 4 x i16> @vdivu_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext
 define <vscale x 4 x i16> @vdivu_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -376,9 +419,14 @@ define <vscale x 8 x i16> @vdivu_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext
 define <vscale x 8 x i16> @vdivu_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 13
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -411,9 +459,14 @@ define <vscale x 16 x i16> @vdivu_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signe
 define <vscale x 16 x i16> @vdivu_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 13
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -446,9 +499,14 @@ define <vscale x 32 x i16> @vdivu_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signe
 define <vscale x 32 x i16> @vdivu_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 13
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -481,9 +539,14 @@ define <vscale x 1 x i32> @vdivu_vx_nxv1i32(<vscale x 1 x i32> %va, i32 signext
 define <vscale x 1 x i32> @vdivu_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 29
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -516,9 +579,14 @@ define <vscale x 2 x i32> @vdivu_vx_nxv2i32(<vscale x 2 x i32> %va, i32 signext
 define <vscale x 2 x i32> @vdivu_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 29
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v8, v25, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -551,9 +619,14 @@ define <vscale x 4 x i32> @vdivu_vx_nxv4i32(<vscale x 4 x i32> %va, i32 signext
 define <vscale x 4 x i32> @vdivu_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 29
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -586,9 +659,14 @@ define <vscale x 8 x i32> @vdivu_vx_nxv8i32(<vscale x 8 x i32> %va, i32 signext
 define <vscale x 8 x i32> @vdivu_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 29
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -621,9 +699,14 @@ define <vscale x 16 x i32> @vdivu_vx_nxv16i32(<vscale x 16 x i32> %va, i32 signe
 define <vscale x 16 x i32> @vdivu_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 29
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -656,9 +739,16 @@ define <vscale x 1 x i64> @vdivu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vdivu_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v8, v26, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -691,9 +781,16 @@ define <vscale x 2 x i64> @vdivu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vdivu_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; CHECK-NEXT:    vmv.v.i v26, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vmseq.vi v0, v26, 1
+; CHECK-NEXT:    vmerge.vvm v8, v28, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -726,9 +823,16 @@ define <vscale x 4 x i64> @vdivu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vdivu_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v12, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v12, v12, a0
+; CHECK-NEXT:    vmseq.vi v0, v28, 1
+; CHECK-NEXT:    vmerge.vvm v8, v12, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -761,9 +865,16 @@ define <vscale x 8 x i64> @vdivu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vdivu_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vdivu_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vdiv.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vmv.v.i v16, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v24, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v24, v24, a0
+; CHECK-NEXT:    vmseq.vi v0, v16, 1
+; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmulh-sdnode-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vmulh-sdnode-rv32.ll
new file mode 100644
index 000000000000..cafa74af1929
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmulh-sdnode-rv32.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-v -verify-machineinstrs < %s | FileCheck %s
+
+; Test that the prepareSREMEqFold optimization doesn't crash on scalable
+; vector types. RVV doesn't have ROTR or ROTL operations so the optimization
+; itself doesn't kick in.
+define <vscale x 4 x i1> @srem_eq_fold_nxv4i8(<vscale x 4 x i8> %va) {
+; CHECK-LABEL: srem_eq_fold_nxv4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, zero, 43
+; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, 6
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v25, v8, v25
+; CHECK-NEXT:    vmseq.vi v0, v25, 0
+; CHECK-NEXT:    ret
+  %head_six = insertelement <vscale x 4 x i8> undef, i8 6, i32 0
+  %splat_six = shufflevector <vscale x 4 x i8> %head_six, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
+  %rem = srem <vscale x 4 x i8> %va, %splat_six
+
+  %cc = icmp eq <vscale x 4 x i8> %rem, zeroinitializer
+  ret <vscale x 4 x i1> %cc
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv32.ll
index f0bf475d5a38..3a5a7b420933 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv32.ll
@@ -26,9 +26,18 @@ define <vscale x 1 x i8> @vrem_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b) {
 define <vscale x 1 x i8> @vrem_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -61,9 +70,18 @@ define <vscale x 2 x i8> @vrem_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b) {
 define <vscale x 2 x i8> @vrem_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -96,9 +114,18 @@ define <vscale x 4 x i8> @vrem_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b) {
 define <vscale x 4 x i8> @vrem_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -131,9 +158,18 @@ define <vscale x 8 x i8> @vrem_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b) {
 define <vscale x 8 x i8> @vrem_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -166,9 +202,18 @@ define <vscale x 16 x i8> @vrem_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %b
 define <vscale x 16 x i8> @vrem_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vrsub.vi v28, v8, 0
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vsrl.vi v28, v26, 7
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -201,9 +246,18 @@ define <vscale x 32 x i8> @vrem_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %b
 define <vscale x 32 x i8> @vrem_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vrsub.vi v12, v8, 0
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    vsra.vi v28, v28, 2
+; CHECK-NEXT:    vsrl.vi v12, v28, 7
+; CHECK-NEXT:    vand.vi v12, v12, -1
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -236,9 +290,18 @@ define <vscale x 64 x i8> @vrem_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %b
 define <vscale x 64 x i8> @vrem_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vrsub.vi v24, v8, 0
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    vsra.vi v16, v16, 2
+; CHECK-NEXT:    vsrl.vi v24, v16, 7
+; CHECK-NEXT:    vand.vi v24, v24, -1
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -271,9 +334,18 @@ define <vscale x 1 x i16> @vrem_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext %
 define <vscale x 1 x i16> @vrem_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -306,9 +378,18 @@ define <vscale x 2 x i16> @vrem_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext %
 define <vscale x 2 x i16> @vrem_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -341,9 +422,18 @@ define <vscale x 4 x i16> @vrem_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext %
 define <vscale x 4 x i16> @vrem_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -376,9 +466,18 @@ define <vscale x 8 x i16> @vrem_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext %
 define <vscale x 8 x i16> @vrem_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v26, v26, 1
+; CHECK-NEXT:    vsrl.vi v28, v26, 15
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -411,9 +510,18 @@ define <vscale x 16 x i16> @vrem_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signex
 define <vscale x 16 x i16> @vrem_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v28, v28, 1
+; CHECK-NEXT:    vsrl.vi v12, v28, 15
+; CHECK-NEXT:    vand.vi v12, v12, -1
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -446,9 +554,18 @@ define <vscale x 32 x i16> @vrem_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signex
 define <vscale x 32 x i16> @vrem_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vadd.vi v16, v16, 0
+; CHECK-NEXT:    vsra.vi v16, v16, 1
+; CHECK-NEXT:    vsrl.vi v24, v16, 15
+; CHECK-NEXT:    vand.vi v24, v24, -1
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -481,9 +598,18 @@ define <vscale x 1 x i32> @vrem_vx_nxv1i32(<vscale x 1 x i32> %va, i32 %b) {
 define <vscale x 1 x i32> @vrem_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vsub.vv v25, v25, v8
+; CHECK-NEXT:    vsra.vi v26, v25, 2
+; CHECK-NEXT:    vsrl.vi v25, v25, 31
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v25, v26, v25
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -516,9 +642,18 @@ define <vscale x 2 x i32> @vrem_vx_nxv2i32(<vscale x 2 x i32> %va, i32 %b) {
 define <vscale x 2 x i32> @vrem_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vsub.vv v25, v25, v8
+; CHECK-NEXT:    vsra.vi v26, v25, 2
+; CHECK-NEXT:    vsrl.vi v25, v25, 31
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v25, v26, v25
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -551,9 +686,18 @@ define <vscale x 4 x i32> @vrem_vx_nxv4i32(<vscale x 4 x i32> %va, i32 %b) {
 define <vscale x 4 x i32> @vrem_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vsub.vv v26, v26, v8
+; CHECK-NEXT:    vsra.vi v28, v26, 2
+; CHECK-NEXT:    vsrl.vi v26, v26, 31
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v26, v28, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -586,9 +730,18 @@ define <vscale x 8 x i32> @vrem_vx_nxv8i32(<vscale x 8 x i32> %va, i32 %b) {
 define <vscale x 8 x i32> @vrem_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vsub.vv v28, v28, v8
+; CHECK-NEXT:    vsra.vi v12, v28, 2
+; CHECK-NEXT:    vsrl.vi v28, v28, 31
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v28, v12, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -621,9 +774,18 @@ define <vscale x 16 x i32> @vrem_vx_nxv16i32(<vscale x 16 x i32> %va, i32 %b) {
 define <vscale x 16 x i32> @vrem_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addi a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vsub.vv v16, v16, v8
+; CHECK-NEXT:    vsra.vi v24, v16, 2
+; CHECK-NEXT:    vsrl.vi v16, v16, 31
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v16, v24, v16
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -663,9 +825,28 @@ define <vscale x 1 x i64> @vrem_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vrem_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v25, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v25, v25, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v26, a1
+; CHECK-NEXT:    vsll.vx v26, v26, a0
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vor.vv v25, v26, v25
+; CHECK-NEXT:    vmulh.vv v25, v8, v25
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v26, v25, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v25, v25, a0
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v25, v26, v25
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -705,9 +886,28 @@ define <vscale x 2 x i64> @vrem_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vrem_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v26, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v26, v26, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v28, a1
+; CHECK-NEXT:    vsll.vx v28, v28, a0
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vor.vv v26, v28, v26
+; CHECK-NEXT:    vmulh.vv v26, v8, v26
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v28, v26, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v26, v28, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -747,9 +947,28 @@ define <vscale x 4 x i64> @vrem_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vrem_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v28, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v28, v28, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v12, a1
+; CHECK-NEXT:    vsll.vx v12, v12, a0
+; CHECK-NEXT:    vsrl.vx v12, v12, a0
+; CHECK-NEXT:    vor.vv v28, v12, v28
+; CHECK-NEXT:    vmulh.vv v28, v8, v28
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v12, v28, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v28, v12, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -789,9 +1008,28 @@ define <vscale x 8 x i64> @vrem_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vrem_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 748983
+; CHECK-NEXT:    addi a0, a0, -586
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmv.v.x v16, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v16, v16, a0
+; CHECK-NEXT:    lui a1, 898779
+; CHECK-NEXT:    addi a1, a1, 1755
+; CHECK-NEXT:    vmv.v.x v24, a1
+; CHECK-NEXT:    vsll.vx v24, v24, a0
+; CHECK-NEXT:    vsrl.vx v24, v24, a0
+; CHECK-NEXT:    vor.vv v16, v24, v16
+; CHECK-NEXT:    vmulh.vv v16, v8, v16
+; CHECK-NEXT:    vadd.vi v16, v16, 0
+; CHECK-NEXT:    vsra.vi v24, v16, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v16, v16, a0
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v16, v24, v16
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv64.ll
index 4635f18deb15..243667128018 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrem-sdnode-rv64.ll
@@ -26,9 +26,18 @@ define <vscale x 1 x i8> @vrem_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b) {
 define <vscale x 1 x i8> @vrem_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -61,9 +70,18 @@ define <vscale x 2 x i8> @vrem_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b) {
 define <vscale x 2 x i8> @vrem_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -96,9 +114,18 @@ define <vscale x 4 x i8> @vrem_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b) {
 define <vscale x 4 x i8> @vrem_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -131,9 +158,18 @@ define <vscale x 8 x i8> @vrem_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b) {
 define <vscale x 8 x i8> @vrem_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 7
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -166,9 +202,18 @@ define <vscale x 16 x i8> @vrem_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %b
 define <vscale x 16 x i8> @vrem_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vrsub.vi v28, v8, 0
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vsrl.vi v28, v26, 7
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -201,9 +246,18 @@ define <vscale x 32 x i8> @vrem_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %b
 define <vscale x 32 x i8> @vrem_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vrsub.vi v12, v8, 0
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    vsra.vi v28, v28, 2
+; CHECK-NEXT:    vsrl.vi v12, v28, 7
+; CHECK-NEXT:    vand.vi v12, v12, -1
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -236,9 +290,18 @@ define <vscale x 64 x i8> @vrem_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %b
 define <vscale x 64 x i8> @vrem_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vrem_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 109
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vrsub.vi v24, v8, 0
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    vsra.vi v16, v16, 2
+; CHECK-NEXT:    vsrl.vi v24, v16, 7
+; CHECK-NEXT:    vand.vi v24, v24, -1
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -271,9 +334,18 @@ define <vscale x 1 x i16> @vrem_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext %
 define <vscale x 1 x i16> @vrem_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -306,9 +378,18 @@ define <vscale x 2 x i16> @vrem_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext %
 define <vscale x 2 x i16> @vrem_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -341,9 +422,18 @@ define <vscale x 4 x i16> @vrem_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext %
 define <vscale x 4 x i16> @vrem_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v25, v25, 1
+; CHECK-NEXT:    vsrl.vi v26, v25, 15
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -376,9 +466,18 @@ define <vscale x 8 x i16> @vrem_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext %
 define <vscale x 8 x i16> @vrem_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v26, v26, 1
+; CHECK-NEXT:    vsrl.vi v28, v26, 15
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -411,9 +510,18 @@ define <vscale x 16 x i16> @vrem_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signex
 define <vscale x 16 x i16> @vrem_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v28, v28, 1
+; CHECK-NEXT:    vsrl.vi v12, v28, 15
+; CHECK-NEXT:    vand.vi v12, v12, -1
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -446,9 +554,18 @@ define <vscale x 32 x i16> @vrem_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signex
 define <vscale x 32 x i16> @vrem_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vrem_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addiw a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vadd.vi v16, v16, 0
+; CHECK-NEXT:    vsra.vi v16, v16, 1
+; CHECK-NEXT:    vsrl.vi v24, v16, 15
+; CHECK-NEXT:    vand.vi v24, v24, -1
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -481,9 +598,19 @@ define <vscale x 1 x i32> @vrem_vx_nxv1i32(<vscale x 1 x i32> %va, i32 signext %
 define <vscale x 1 x i32> @vrem_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 31
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -516,9 +643,19 @@ define <vscale x 2 x i32> @vrem_vx_nxv2i32(<vscale x 2 x i32> %va, i32 signext %
 define <vscale x 2 x i32> @vrem_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vrsub.vi v26, v8, 0
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vsra.vi v25, v25, 2
+; CHECK-NEXT:    vsrl.vi v26, v25, 31
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -551,9 +688,19 @@ define <vscale x 4 x i32> @vrem_vx_nxv4i32(<vscale x 4 x i32> %va, i32 signext %
 define <vscale x 4 x i32> @vrem_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vrsub.vi v28, v8, 0
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vsra.vi v26, v26, 2
+; CHECK-NEXT:    vsrl.vi v28, v26, 31
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -586,9 +733,19 @@ define <vscale x 8 x i32> @vrem_vx_nxv8i32(<vscale x 8 x i32> %va, i32 signext %
 define <vscale x 8 x i32> @vrem_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vrsub.vi v12, v8, 0
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    vsra.vi v28, v28, 2
+; CHECK-NEXT:    vsrl.vi v12, v28, 31
+; CHECK-NEXT:    vand.vi v12, v12, -1
+; CHECK-NEXT:    vadd.vv v28, v28, v12
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -621,9 +778,19 @@ define <vscale x 16 x i32> @vrem_vx_nxv16i32(<vscale x 16 x i32> %va, i32 signex
 define <vscale x 16 x i32> @vrem_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vrem_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 449390
+; CHECK-NEXT:    addiw a0, a0, -1171
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vrsub.vi v24, v8, 0
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    vsra.vi v16, v16, 2
+; CHECK-NEXT:    vsrl.vi v24, v16, 31
+; CHECK-NEXT:    vand.vi v24, v24, -1
+; CHECK-NEXT:    vadd.vv v16, v16, v24
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -656,9 +823,25 @@ define <vscale x 1 x i64> @vrem_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vrem_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v25, v8, a0
+; CHECK-NEXT:    vadd.vi v25, v25, 0
+; CHECK-NEXT:    vsra.vi v26, v25, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v25, v25, a0
+; CHECK-NEXT:    vand.vi v25, v25, -1
+; CHECK-NEXT:    vadd.vv v25, v26, v25
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -691,9 +874,25 @@ define <vscale x 2 x i64> @vrem_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vrem_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v26, v8, a0
+; CHECK-NEXT:    vadd.vi v26, v26, 0
+; CHECK-NEXT:    vsra.vi v28, v26, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vand.vi v26, v26, -1
+; CHECK-NEXT:    vadd.vv v26, v28, v26
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -726,9 +925,25 @@ define <vscale x 4 x i64> @vrem_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vrem_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v28, v8, a0
+; CHECK-NEXT:    vadd.vi v28, v28, 0
+; CHECK-NEXT:    vsra.vi v12, v28, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vand.vi v28, v28, -1
+; CHECK-NEXT:    vadd.vv v28, v12, v28
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -761,9 +976,25 @@ define <vscale x 8 x i64> @vrem_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vrem_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vrem_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 1029851
+; CHECK-NEXT:    addiw a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
+; CHECK-NEXT:    slli a0, a0, 12
+; CHECK-NEXT:    addi a0, a0, 1755
 ; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vrem.vx v8, v8, a0
+; CHECK-NEXT:    vmulh.vx v16, v8, a0
+; CHECK-NEXT:    vadd.vi v16, v16, 0
+; CHECK-NEXT:    vsra.vi v24, v16, 1
+; CHECK-NEXT:    addi a0, zero, 63
+; CHECK-NEXT:    vsrl.vx v16, v16, a0
+; CHECK-NEXT:    vand.vi v16, v16, -1
+; CHECK-NEXT:    vadd.vv v16, v24, v16
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv32.ll
index e669598464b5..6fba6c90ac46 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv32.ll
@@ -26,9 +26,16 @@ define <vscale x 1 x i8> @vremu_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b)
 define <vscale x 1 x i8> @vremu_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -61,9 +68,16 @@ define <vscale x 2 x i8> @vremu_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b)
 define <vscale x 2 x i8> @vremu_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -96,9 +110,16 @@ define <vscale x 4 x i8> @vremu_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b)
 define <vscale x 4 x i8> @vremu_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -131,9 +152,16 @@ define <vscale x 8 x i8> @vremu_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b)
 define <vscale x 8 x i8> @vremu_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -166,9 +194,16 @@ define <vscale x 16 x i8> @vremu_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %
 define <vscale x 16 x i8> @vremu_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 5
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v26, v26, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -201,9 +236,16 @@ define <vscale x 32 x i8> @vremu_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %
 define <vscale x 32 x i8> @vremu_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 5
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v28, v28, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -236,9 +278,16 @@ define <vscale x 64 x i8> @vremu_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %
 define <vscale x 64 x i8> @vremu_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 5
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -271,9 +320,17 @@ define <vscale x 1 x i16> @vremu_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext
 define <vscale x 1 x i16> @vremu_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -306,9 +363,17 @@ define <vscale x 2 x i16> @vremu_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext
 define <vscale x 2 x i16> @vremu_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -341,9 +406,17 @@ define <vscale x 4 x i16> @vremu_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext
 define <vscale x 4 x i16> @vremu_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -376,9 +449,17 @@ define <vscale x 8 x i16> @vremu_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext
 define <vscale x 8 x i16> @vremu_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 13
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v26, v26, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -411,9 +492,17 @@ define <vscale x 16 x i16> @vremu_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signe
 define <vscale x 16 x i16> @vremu_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 13
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v28, v28, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -446,9 +535,17 @@ define <vscale x 32 x i16> @vremu_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signe
 define <vscale x 32 x i16> @vremu_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 13
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -481,9 +578,17 @@ define <vscale x 1 x i32> @vremu_vx_nxv1i32(<vscale x 1 x i32> %va, i32 %b) {
 define <vscale x 1 x i32> @vremu_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,mf2,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 29
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v25, v26, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -516,9 +621,17 @@ define <vscale x 2 x i32> @vremu_vx_nxv2i32(<vscale x 2 x i32> %va, i32 %b) {
 define <vscale x 2 x i32> @vremu_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 29
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v25, v26, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -551,9 +664,17 @@ define <vscale x 4 x i32> @vremu_vx_nxv4i32(<vscale x 4 x i32> %va, i32 %b) {
 define <vscale x 4 x i32> @vremu_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
+; CHECK-NEXT:    vmv.v.i v26, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 29
+; CHECK-NEXT:    vmseq.vi v0, v26, 1
+; CHECK-NEXT:    vmerge.vvm v26, v28, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -586,9 +707,17 @@ define <vscale x 8 x i32> @vremu_vx_nxv8i32(<vscale x 8 x i32> %va, i32 %b) {
 define <vscale x 8 x i32> @vremu_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v12, v8, a0
+; CHECK-NEXT:    vsrl.vi v12, v12, 29
+; CHECK-NEXT:    vmseq.vi v0, v28, 1
+; CHECK-NEXT:    vmerge.vvm v28, v12, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -621,9 +750,17 @@ define <vscale x 16 x i32> @vremu_vx_nxv16i32(<vscale x 16 x i32> %va, i32 %b) {
 define <vscale x 16 x i32> @vremu_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
+; CHECK-NEXT:    vmv.v.i v16, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v24, v8, a0
+; CHECK-NEXT:    vsrl.vi v24, v24, 29
+; CHECK-NEXT:    vmseq.vi v0, v16, 1
+; CHECK-NEXT:    vmerge.vvm v16, v24, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -663,9 +800,24 @@ define <vscale x 1 x i64> @vremu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vremu_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v26, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v26, v26, a0
+; CHECK-NEXT:    vmv.v.i v27, 1
+; CHECK-NEXT:    vsll.vx v27, v27, a0
+; CHECK-NEXT:    vsrl.vx v27, v27, a0
+; CHECK-NEXT:    vor.vv v26, v27, v26
+; CHECK-NEXT:    vmulhu.vv v26, v8, v26
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v25, v26, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -705,9 +857,24 @@ define <vscale x 2 x i64> @vremu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vremu_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; CHECK-NEXT:    vmv.v.i v26, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v28, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v28, v28, a0
+; CHECK-NEXT:    vmv.v.i v30, 1
+; CHECK-NEXT:    vsll.vx v30, v30, a0
+; CHECK-NEXT:    vsrl.vx v30, v30, a0
+; CHECK-NEXT:    vor.vv v28, v30, v28
+; CHECK-NEXT:    vmulhu.vv v28, v8, v28
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vmseq.vi v0, v26, 1
+; CHECK-NEXT:    vmerge.vvm v26, v28, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -747,9 +914,24 @@ define <vscale x 4 x i64> @vremu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vremu_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v12, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v12, v12, a0
+; CHECK-NEXT:    vmv.v.i v16, 1
+; CHECK-NEXT:    vsll.vx v16, v16, a0
+; CHECK-NEXT:    vsrl.vx v16, v16, a0
+; CHECK-NEXT:    vor.vv v12, v16, v12
+; CHECK-NEXT:    vmulhu.vv v12, v8, v12
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v12, v12, a0
+; CHECK-NEXT:    vmseq.vi v0, v28, 1
+; CHECK-NEXT:    vmerge.vvm v28, v12, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -789,9 +971,24 @@ define <vscale x 8 x i64> @vremu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vremu_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vmv.v.i v16, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    vmv.v.x v24, a0
+; CHECK-NEXT:    addi a0, zero, 32
+; CHECK-NEXT:    vsll.vx v24, v24, a0
+; CHECK-NEXT:    vmv.v.i v0, 1
+; CHECK-NEXT:    vsll.vx v0, v0, a0
+; CHECK-NEXT:    vsrl.vx v0, v0, a0
+; CHECK-NEXT:    vor.vv v24, v0, v24
+; CHECK-NEXT:    vmulhu.vv v24, v8, v24
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v24, v24, a0
+; CHECK-NEXT:    vmseq.vi v0, v16, 1
+; CHECK-NEXT:    vmerge.vvm v16, v24, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv64.ll
index 304a8d4ed5f5..5e96d40005df 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vremu-sdnode-rv64.ll
@@ -26,9 +26,16 @@ define <vscale x 1 x i8> @vremu_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b)
 define <vscale x 1 x i8> @vremu_vi_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -61,9 +68,16 @@ define <vscale x 2 x i8> @vremu_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b)
 define <vscale x 2 x i8> @vremu_vi_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -96,9 +110,16 @@ define <vscale x 4 x i8> @vremu_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b)
 define <vscale x 4 x i8> @vremu_vi_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,mf2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -131,9 +152,16 @@ define <vscale x 8 x i8> @vremu_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b)
 define <vscale x 8 x i8> @vremu_vi_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 5
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
@@ -166,9 +194,16 @@ define <vscale x 16 x i8> @vremu_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %
 define <vscale x 16 x i8> @vremu_vi_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 5
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v26, v26, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 16 x i8> %head, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
@@ -201,9 +236,16 @@ define <vscale x 32 x i8> @vremu_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %
 define <vscale x 32 x i8> @vremu_vi_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 5
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v28, v28, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 32 x i8> %head, <vscale x 32 x i8> undef, <vscale x 32 x i32> zeroinitializer
@@ -236,9 +278,16 @@ define <vscale x 64 x i8> @vremu_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %
 define <vscale x 64 x i8> @vremu_vi_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vremu_vi_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    addi a0, zero, 33
 ; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 5
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 64 x i8> undef, i8 -7, i32 0
   %splat = shufflevector <vscale x 64 x i8> %head, <vscale x 64 x i8> undef, <vscale x 64 x i32> zeroinitializer
@@ -271,9 +320,17 @@ define <vscale x 1 x i16> @vremu_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext
 define <vscale x 1 x i16> @vremu_vi_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -306,9 +363,17 @@ define <vscale x 2 x i16> @vremu_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext
 define <vscale x 2 x i16> @vremu_vi_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,mf2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -341,9 +406,17 @@ define <vscale x 4 x i16> @vremu_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext
 define <vscale x 4 x i16> @vremu_vi_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 13
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -376,9 +449,17 @@ define <vscale x 8 x i16> @vremu_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext
 define <vscale x 8 x i16> @vremu_vi_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 13
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v26, v26, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -411,9 +492,17 @@ define <vscale x 16 x i16> @vremu_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signe
 define <vscale x 16 x i16> @vremu_vi_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 13
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v28, v28, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 16 x i16> %head, <vscale x 16 x i16> undef, <vscale x 16 x i32> zeroinitializer
@@ -446,9 +535,17 @@ define <vscale x 32 x i16> @vremu_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signe
 define <vscale x 32 x i16> @vremu_vi_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vremu_vi_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 2
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 13
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i16> undef, i16 -7, i32 0
   %splat = shufflevector <vscale x 32 x i16> %head, <vscale x 32 x i16> undef, <vscale x 32 x i32> zeroinitializer
@@ -481,9 +578,17 @@ define <vscale x 1 x i32> @vremu_vx_nxv1i32(<vscale x 1 x i32> %va, i32 signext
 define <vscale x 1 x i32> @vremu_vi_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,mf2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 29
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 1 x i32> %head, <vscale x 1 x i32> undef, <vscale x 1 x i32> zeroinitializer
@@ -516,9 +621,17 @@ define <vscale x 2 x i32> @vremu_vx_nxv2i32(<vscale x 2 x i32> %va, i32 signext
 define <vscale x 2 x i32> @vremu_vi_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v25, v8, a0
+; CHECK-NEXT:    vsrl.vi v25, v25, 29
+; CHECK-NEXT:    vmv.v.i v26, 1
+; CHECK-NEXT:    vmseq.vi v0, v26, -7
+; CHECK-NEXT:    vmerge.vvm v25, v25, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 2 x i32> %head, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
@@ -551,9 +664,17 @@ define <vscale x 4 x i32> @vremu_vx_nxv4i32(<vscale x 4 x i32> %va, i32 signext
 define <vscale x 4 x i32> @vremu_vi_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    vsrl.vi v26, v26, 29
+; CHECK-NEXT:    vmv.v.i v28, 1
+; CHECK-NEXT:    vmseq.vi v0, v28, -7
+; CHECK-NEXT:    vmerge.vvm v26, v26, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 4 x i32> %head, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -586,9 +707,17 @@ define <vscale x 8 x i32> @vremu_vx_nxv8i32(<vscale x 8 x i32> %va, i32 signext
 define <vscale x 8 x i32> @vremu_vi_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    vsrl.vi v28, v28, 29
+; CHECK-NEXT:    vmv.v.i v12, 1
+; CHECK-NEXT:    vmseq.vi v0, v12, -7
+; CHECK-NEXT:    vmerge.vvm v28, v28, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> undef, <vscale x 8 x i32> zeroinitializer
@@ -621,9 +750,17 @@ define <vscale x 16 x i32> @vremu_vx_nxv16i32(<vscale x 16 x i32> %va, i32 signe
 define <vscale x 16 x i32> @vremu_vi_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vremu_vi_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    lui a0, 131072
+; CHECK-NEXT:    addiw a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmulhu.vx v16, v8, a0
+; CHECK-NEXT:    vsrl.vi v16, v16, 29
+; CHECK-NEXT:    vmv.v.i v24, 1
+; CHECK-NEXT:    vmseq.vi v0, v24, -7
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    addi a0, zero, -7
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i32> undef, i32 -7, i32 0
   %splat = shufflevector <vscale x 16 x i32> %head, <vscale x 16 x i32> undef, <vscale x 16 x i32> zeroinitializer
@@ -656,9 +793,19 @@ define <vscale x 1 x i64> @vremu_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vremu_vi_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv1i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vmv.v.i v25, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v26, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v26, v26, a0
+; CHECK-NEXT:    vmseq.vi v0, v25, 1
+; CHECK-NEXT:    vmerge.vvm v25, v26, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v25, v25, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v25
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 1 x i64> %head, <vscale x 1 x i64> undef, <vscale x 1 x i32> zeroinitializer
@@ -691,9 +838,19 @@ define <vscale x 2 x i64> @vremu_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vremu_vi_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv2i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
+; CHECK-NEXT:    vmv.v.i v26, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v28, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v28, v28, a0
+; CHECK-NEXT:    vmseq.vi v0, v26, 1
+; CHECK-NEXT:    vmerge.vvm v26, v28, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v26, v26, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v26
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 2 x i64> %head, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
@@ -726,9 +883,19 @@ define <vscale x 4 x i64> @vremu_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vremu_vi_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv4i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
+; CHECK-NEXT:    vmv.v.i v28, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v12, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v12, v12, a0
+; CHECK-NEXT:    vmseq.vi v0, v28, 1
+; CHECK-NEXT:    vmerge.vvm v28, v12, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v28, v28, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v28
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 4 x i64> %head, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
@@ -761,9 +928,19 @@ define <vscale x 8 x i64> @vremu_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vremu_vi_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vremu_vi_nxv8i64_0:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vmv.v.i v16, -7
+; CHECK-NEXT:    addi a0, zero, 1
+; CHECK-NEXT:    slli a0, a0, 61
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    vmulhu.vx v24, v8, a0
+; CHECK-NEXT:    addi a0, zero, 61
+; CHECK-NEXT:    vsrl.vx v24, v24, a0
+; CHECK-NEXT:    vmseq.vi v0, v16, 1
+; CHECK-NEXT:    vmerge.vvm v16, v24, v8, v0
 ; CHECK-NEXT:    addi a0, zero, -7
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
-; CHECK-NEXT:    vremu.vx v8, v8, a0
+; CHECK-NEXT:    vmul.vx v16, v16, a0
+; CHECK-NEXT:    vsub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> undef, i64 -7, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> undef, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/X86/haddsub-2.ll b/llvm/test/CodeGen/X86/haddsub-2.ll
index 82fd7a2699a5..c022cd4e072b 100644
--- a/llvm/test/CodeGen/X86/haddsub-2.ll
+++ b/llvm/test/CodeGen/X86/haddsub-2.ll
@@ -587,9 +587,8 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
 ;
 ; AVX2-LABEL: avx2_vphadd_d_test:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vphaddd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    retq
   %vecext = extractelement <8 x i32> %A, i32 0
   %vecext1 = extractelement <8 x i32> %A, i32 1
@@ -743,9 +742,8 @@ define <16 x i16> @avx2_vphadd_w_test(<16 x i16> %a, <16 x i16> %b) nounwind {
 ;
 ; AVX2-LABEL: avx2_vphadd_w_test:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vphaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    retq
   %vecext = extractelement <16 x i16> %a, i32 0
   %vecext1 = extractelement <16 x i16> %a, i32 1
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index a4923e04c090..6107e8c0ca5f 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -5138,9 +5138,8 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm1
 ; AVX2-NEXT:    vpmovmskb %ymm1, %eax
 ; AVX2-NEXT:    notl %eax
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index e944776cea95..98335d362f95 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -626,16 +626,14 @@ define <8 x i32> @var_shuffle_v8i32_from_v4i32(<4 x i32> %v, <8 x i32> %indices)
 ; XOP-LABEL: var_shuffle_v8i32_from_v4i32:
 ; XOP:       # %bb.0: # %entry
 ; XOP-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; XOP-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
 ; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT:    vpermil2ps $0, %ymm1, %ymm2, %ymm0, %ymm0
+; XOP-NEXT:    vpermil2ps $0, %ymm1, %ymm0, %ymm0, %ymm0
 ; XOP-NEXT:    retq
 ;
 ; AVX1-LABEL: var_shuffle_v8i32_from_v4i32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vpermilps %ymm1, %ymm2, %ymm2
+; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm3
@@ -1049,16 +1047,14 @@ define <8 x float> @var_shuffle_v8f32_from_v4f32(<4 x float> %v, <8 x i32> %indi
 ; XOP-LABEL: var_shuffle_v8f32_from_v4f32:
 ; XOP:       # %bb.0: # %entry
 ; XOP-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; XOP-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
 ; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; XOP-NEXT:    vpermil2ps $0, %ymm1, %ymm2, %ymm0, %ymm0
+; XOP-NEXT:    vpermil2ps $0, %ymm1, %ymm0, %ymm0, %ymm0
 ; XOP-NEXT:    retq
 ;
 ; AVX1-LABEL: var_shuffle_v8f32_from_v4f32:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
-; AVX1-NEXT:    vpermilps %ymm1, %ymm2, %ymm2
+; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vpcmpgtd {{.*}}(%rip), %xmm1, %xmm3
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index e6b895692cc0..2958f7e96cb5 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -717,9 +717,8 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    cmpl $-1, %eax
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index a5458d5264bb..283939f86a94 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -700,9 +700,8 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    testl %eax, %eax
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index efb45ee36f9f..a60dbcd7480a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -756,9 +756,8 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX2-NEXT:    movl %eax, %ecx
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index ac1144818b07..d8e4c0085970 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -5964,7 +5964,7 @@ define <16 x i16> @shuffle_v16i16_21_22_23_00_01_02_03_12_29_30_31_08_09_10_11_1
 ; AVX2-LABEL: shuffle_v16i16_21_22_23_00_01_02_03_12_29_30_31_08_09_10_11_12:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
 ; AVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
 ; AVX2-NEXT:    retq
@@ -5988,7 +5988,7 @@ define <16 x i16> @shuffle_v16i16_21_22_23_00_01_02_03_12_29_30_31_08_09_10_11_1
 ; XOPAVX2-LABEL: shuffle_v16i16_21_22_23_00_01_02_03_12_29_30_31_08_09_10_11_12:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
-; XOPAVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
+; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
 ; XOPAVX2-NEXT:    retq
@@ -6146,7 +6146,7 @@ define <16 x i16> @shuffle_v16i16_19_20_21_22_23_00_01_10_27_28_29_30_31_08_09_1
 ; AVX2-LABEL: shuffle_v16i16_19_20_21_22_23_00_01_10_27_28_29_30_31_08_09_10:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4,5,6,7,8,9],ymm0[10],ymm1[11,12,13,14,15]
 ; AVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
 ; AVX2-NEXT:    retq
@@ -6171,7 +6171,7 @@ define <16 x i16> @shuffle_v16i16_19_20_21_22_23_00_01_10_27_28_29_30_31_08_09_1
 ; XOPAVX2-LABEL: shuffle_v16i16_19_20_21_22_23_00_01_10_27_28_29_30_31_08_09_10:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
-; XOPAVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
+; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4,5,6,7,8,9],ymm0[10],ymm1[11,12,13,14,15]
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
 ; XOPAVX2-NEXT:    retq
@@ -6329,7 +6329,7 @@ define <16 x i16> @shuffle_v16i16_03_04_05_06_07_16_17_26_11_12_13_14_15_24_25_2
 ; AVX2-LABEL: shuffle_v16i16_03_04_05_06_07_16_17_26_11_12_13_14_15_24_25_26:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[2,3]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7,8,9],ymm1[10],ymm0[11,12,13,14,15]
 ; AVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
 ; AVX2-NEXT:    retq
@@ -6353,7 +6353,7 @@ define <16 x i16> @shuffle_v16i16_03_04_05_06_07_16_17_26_11_12_13_14_15_24_25_2
 ; XOPAVX2-LABEL: shuffle_v16i16_03_04_05_06_07_16_17_26_11_12_13_14_15_24_25_26:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
-; XOPAVX2-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[2,3]
+; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7,8,9],ymm1[10],ymm0[11,12,13,14,15]
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20,21]
 ; XOPAVX2-NEXT:    retq
@@ -6407,7 +6407,7 @@ define <16 x i16> @shuffle_v16i16_05_06_07_16_17_18_19_28_13_14_15_24_25_26_27_2
 ; AVX2-LABEL: shuffle_v16i16_05_06_07_16_17_18_19_28_13_14_15_24_25_26_27_28:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[2,3]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
 ; AVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
 ; AVX2-NEXT:    retq
@@ -6432,7 +6432,7 @@ define <16 x i16> @shuffle_v16i16_05_06_07_16_17_18_19_28_13_14_15_24_25_26_27_2
 ; XOPAVX2-LABEL: shuffle_v16i16_05_06_07_16_17_18_19_28_13_14_15_24_25_26_27_28:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
-; XOPAVX2-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[2,3]
+; XOPAVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
 ; XOPAVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
 ; XOPAVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 8d5bd48119eb..6a103f642c76 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -1306,9 +1306,8 @@ define void @trunc32i16_32i8(<32 x i16> %a) {
 ; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],ymm1[2,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rax)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/xray-custom-log.ll b/llvm/test/CodeGen/X86/xray-custom-log.ll
index f53e6c676c3b..f0d882ddbac6 100644
--- a/llvm/test/CodeGen/X86/xray-custom-log.ll
+++ b/llvm/test/CodeGen/X86/xray-custom-log.ll
@@ -1,13 +1,15 @@
-; RUN: llc -verify-machineinstrs -debug-entry-values -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -debug-entry-values -filetype=asm -o - \
-; RUN:     -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC
+; RUN: llc -verify-machineinstrs -mtriple=x86_64 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=x86_64 -relocation-model=pic < %s | FileCheck %s --check-prefix=PIC
 
-define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
+; RUN: llc -mtriple=x86_64 -filetype=obj %s -o %t
+; RUN: llvm-dwarfdump %t | FileCheck %s --check-prefix=DBG
+
+define i32 @customevent() nounwind "function-instrument"="xray-always" !dbg !1 {
     %eventptr = alloca i8
     %eventsize = alloca i32
     store i32 3, i32* %eventsize
     %val = load i32, i32* %eventsize
-    call void @llvm.xray.customevent(i8* %eventptr, i32 %val)
+    call void @llvm.xray.customevent(i8* %eventptr, i32 %val), !dbg !8
     ; CHECK-LABEL: Lxray_event_sled_0:
     ; CHECK:       .byte 0xeb, 0x0f
     ; CHECK-NEXT:  pushq %rdi
@@ -33,4 +35,72 @@ define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
 ; CHECK-LABEL: Lxray_sleds_start0:
 ; CHECK:       .quad {{.*}}xray_event_sled_0
 
+define i32 @typedevent() nounwind "function-instrument"="xray-always" !dbg !2 {
+    %eventptr = alloca i8
+    %eventsize = alloca i32
+    %eventtype = alloca i16
+    store i16 6, i16* %eventtype
+    %type = load i16, i16* %eventtype
+    store i32 3, i32* %eventsize
+    %val = load i32, i32* %eventsize
+    call void @llvm.xray.typedevent(i16 %type, i8* %eventptr, i32 %val), !dbg !9
+    ; CHECK-LABEL: Lxray_typed_event_sled_0:
+    ; CHECK:       .byte 0xeb, 0x14
+    ; CHECK-NEXT:  pushq %rdi
+    ; CHECK-NEXT:  pushq %rsi
+    ; CHECK-NEXT:  pushq %rdx
+    ; CHECK-NEXT:  movq %rdx, %rdi
+    ; CHECK-NEXT:  movq %rcx, %rsi
+    ; CHECK-NEXT:  movq %rax, %rdx
+    ; CHECK-NEXT:  callq __xray_TypedEvent
+    ; CHECK-NEXT:  popq %rdx
+    ; CHECK-NEXT:  popq %rsi
+    ; CHECK-NEXT:  popq %rdi
+
+    ; PIC-LABEL: Lxray_typed_event_sled_0:
+    ; PIC:       .byte 0xeb, 0x14
+    ; PIC-NEXT:  pushq %rdi
+    ; PIC-NEXT:  pushq %rsi
+    ; PIC-NEXT:  pushq %rdx
+    ; PIC-NEXT:  movq %rdx, %rdi
+    ; PIC-NEXT:  movq %rcx, %rsi
+    ; PIC-NEXT:  movq %rax, %rdx
+    ; PIC-NEXT:  callq __xray_TypedEvent@PLT
+    ; PIC-NEXT:  popq %rdx
+    ; PIC-NEXT:  popq %rsi
+    ; PIC-NEXT:  popq %rdi
+    ret i32 0
+}
+; CHECK-LABEL: xray_instr_map
+; CHECK-LABEL: Lxray_sleds_start1:
+; CHECK:       .quad {{.*}}xray_typed_event_sled_0
+
 declare void @llvm.xray.customevent(i8*, i32)
+declare void @llvm.xray.typedevent(i16, i8*, i32)
+
+;; Construct call site entries for PATCHABLE_EVENT_CALL.
+; DBG:     DW_TAG_subprogram
+; DBG:       DW_TAG_call_site
+; DBG-NEXT:    DW_AT_call_target (DW_OP_reg{{.*}})
+; DBG-NEXT:    DW_AT_call_return_pc
+
+; DBG:     DW_TAG_subprogram
+; DBG:       DW_TAG_call_site
+; DBG-NEXT:    DW_AT_call_target (DW_OP_reg{{.*}})
+; DBG-NEXT:    DW_AT_call_return_pc
+
+!llvm.dbg.cu = !{!7}
+!llvm.module.flags = !{!10, !11}
+
+!1 = distinct !DISubprogram(name: "customevent", scope: !3, file: !3, line: 1, type: !4, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7)
+!2 = distinct !DISubprogram(name: "typedevent", scope: !3, file: !3, line: 3, type: !4, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7)
+!3 = !DIFile(filename: "a.c", directory: "/tmp")
+!4 = !DISubroutineType(types: !5)
+!5 = !{!6}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 12.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!8 = !DILocation(line: 2, column: 3, scope: !1)
+!9 = !DILocation(line: 4, column: 3, scope: !2)
+
+!10 = !{i32 7, !"Dwarf Version", i32 5}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/CodeGen/X86/xray-typed-event-log.ll b/llvm/test/CodeGen/X86/xray-typed-event-log.ll
deleted file mode 100644
index aa7e19947f95..000000000000
--- a/llvm/test/CodeGen/X86/xray-typed-event-log.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: llc -verify-machineinstrs -debug-entry-values -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -debug-entry-values -filetype=asm -o - -mtriple=x86_64-unknown-linux-gnu \
-; RUN:    -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC
-
-define i32 @fn() nounwind noinline uwtable "function-instrument"="xray-always" {
-    %eventptr = alloca i8
-    %eventsize = alloca i32
-    %eventtype = alloca i16
-    store i16 6, i16* %eventtype
-    %type = load i16, i16* %eventtype
-    store i32 3, i32* %eventsize
-    %val = load i32, i32* %eventsize
-    call void @llvm.xray.typedevent(i16 %type, i8* %eventptr, i32 %val)
-    ; CHECK-LABEL: Lxray_typed_event_sled_0:
-    ; CHECK:       .byte 0xeb, 0x14
-    ; CHECK-NEXT:  pushq %rdi
-    ; CHECK-NEXT:  pushq %rsi
-    ; CHECK-NEXT:  pushq %rdx
-    ; CHECK-NEXT:  movq %rdx, %rdi
-    ; CHECK-NEXT:  movq %rcx, %rsi
-    ; CHECK-NEXT:  movq %rax, %rdx
-    ; CHECK-NEXT:  callq __xray_TypedEvent
-    ; CHECK-NEXT:  popq %rdx
-    ; CHECK-NEXT:  popq %rsi
-    ; CHECK-NEXT:  popq %rdi
-
-    ; PIC-LABEL: Lxray_typed_event_sled_0:
-    ; PIC:       .byte 0xeb, 0x14
-    ; PIC-NEXT:  pushq %rdi
-    ; PIC-NEXT:  pushq %rsi
-    ; PIC-NEXT:  pushq %rdx
-    ; PIC-NEXT:  movq %rdx, %rdi
-    ; PIC-NEXT:  movq %rcx, %rsi
-    ; PIC-NEXT:  movq %rax, %rdx
-    ; PIC-NEXT:  callq __xray_TypedEvent@PLT
-    ; PIC-NEXT:  popq %rdx
-    ; PIC-NEXT:  popq %rsi
-    ; PIC-NEXT:  popq %rdi
-    ret i32 0
-}
-; CHECK-LABEL: xray_instr_map
-; CHECK-LABEL: Lxray_sleds_start0:
-; CHECK:       .quad {{.*}}xray_typed_event_sled_0
-
-declare void @llvm.xray.typedevent(i16, i8*, i32)
diff --git a/llvm/test/DebugInfo/symbolize-missing-file.test b/llvm/test/DebugInfo/symbolize-missing-file.test
index 8665b2ff1dd2..2762af4ff5da 100644
--- a/llvm/test/DebugInfo/symbolize-missing-file.test
+++ b/llvm/test/DebugInfo/symbolize-missing-file.test
@@ -1,3 +1,3 @@
 RUN: llvm-symbolizer --obj=unexisting-file 0x1234 2>&1 | FileCheck %s
 
-CHECK: LLVMSymbolizer: error reading file: {{.*}}{{[Nn]}}o such file or directory
+CHECK: LLVMSymbolizer: error reading file: {{[Nn]}}o such file or directory
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_basic.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_basic.s
deleted file mode 100644
index fe00d23f6f66..000000000000
--- a/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_basic.s
+++ /dev/null
@@ -1,115 +0,0 @@
-# REQUIRES: asserts
-# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent \
-# RUN:     -filetype=obj -o %t %s
-# RUN: llvm-jitlink -debug-only=jitlink -define-abs bar=0x01 -noexec %t 2>&1 | \
-# RUN:     FileCheck %s
-#
-# Check that a basic .eh-frame section is recognized and parsed. We
-# Expect to see two FDEs with corresponding keep-alive edges.
-#
-# CHECK: Adding keep-alive edge from target at {{.*}} to FDE at
-# CHECK: Adding keep-alive edge from target at {{.*}} to FDE at
-
-	.text
-	.file	"exceptions.cpp"
-	.globl	foo
-	.p2align	4, 0x90
-	.type	foo,@function
-foo:
-	.cfi_startproc
-
-	pushq	%rax
-	.cfi_def_cfa_offset 16
-	movl	$4, %edi
-	callq	__cxa_allocate_exception@PLT
-	movl	$1, (%rax)
-	movq	_ZTIi@GOTPCREL(%rip), %rsi
-	movq	%rax, %rdi
-	xorl	%edx, %edx
-	callq	__cxa_throw@PLT
-.Lfunc_end0:
-	.size	foo, .Lfunc_end0-foo
-	.cfi_endproc
-
-	.globl	main
-	.p2align	4, 0x90
-	.type	main,@function
-main:
-.Lfunc_begin0:
-	.cfi_startproc
-	.cfi_personality 155, DW.ref.__gxx_personality_v0
-	.cfi_lsda 27, .Lexception0
-
-	pushq	%rbx
-	.cfi_def_cfa_offset 16
-	.cfi_offset %rbx, -16
-	xorl	%ebx, %ebx
-.Ltmp0:
-	callq	bar@PLT
-.Ltmp1:
-
-	movl	%ebx, %eax
-	popq	%rbx
-	.cfi_def_cfa_offset 8
-	retq
-.LBB1_1:
-	.cfi_def_cfa_offset 16
-.Ltmp2:
-	movq	%rax, %rdi
-	callq	__cxa_begin_catch@PLT
-	callq	__cxa_end_catch@PLT
-	movl	$1, %ebx
-	movl	%ebx, %eax
-	popq	%rbx
-	.cfi_def_cfa_offset 8
-	retq
-.Lfunc_end1:
-	.size	main, .Lfunc_end1-main
-	.cfi_endproc
-	.section	.gcc_except_table,"a",@progbits
-	.p2align	2
-GCC_except_table1:
-.Lexception0:
-	.byte	255
-	.byte	156
-	.uleb128 .Lttbase0-.Lttbaseref0
-.Lttbaseref0:
-	.byte	1
-	.uleb128 .Lcst_end0-.Lcst_begin0
-.Lcst_begin0:
-	.uleb128 .Ltmp0-.Lfunc_begin0
-	.uleb128 .Ltmp1-.Ltmp0
-	.uleb128 .Ltmp2-.Lfunc_begin0
-	.byte	1
-	.uleb128 .Ltmp1-.Lfunc_begin0
-	.uleb128 .Lfunc_end1-.Ltmp1
-	.byte	0
-	.byte	0
-.Lcst_end0:
-	.byte	1
-
-	.byte	0
-	.p2align	2
-
-.Ltmp3:
-	.quad	.L_ZTIi.DW.stub-.Ltmp3
-.Lttbase0:
-	.p2align	2
-
-	.data
-	.p2align	3
-.L_ZTIi.DW.stub:
-	.quad	_ZTIi
-	.hidden	DW.ref.__gxx_personality_v0
-	.weak	DW.ref.__gxx_personality_v0
-	.section	.data.DW.ref.__gxx_personality_v0,"aGw",@progbits,DW.ref.__gxx_personality_v0,comdat
-	.p2align	3
-	.type	DW.ref.__gxx_personality_v0,@object
-	.size	DW.ref.__gxx_personality_v0, 8
-DW.ref.__gxx_personality_v0:
-	.quad	__gxx_personality_v0
-	.ident	"clang version 12.0.0 (git@github.com:llvm/llvm-project.git afd483e57d166418e94a65bd9716e7dc4c114eed)"
-	.section	".note.GNU-stack","",@progbits
-	.addrsig
-	.addrsig_sym __gxx_personality_v0
-	.addrsig_sym _ZTIi
diff --git a/llvm/test/MC/AMDGPU/gfx10_err_pos.s b/llvm/test/MC/AMDGPU/gfx10_err_pos.s
index cb4f9ae91153..b0cf97921daa 100644
--- a/llvm/test/MC/AMDGPU/gfx10_err_pos.s
+++ b/llvm/test/MC/AMDGPU/gfx10_err_pos.s
@@ -483,6 +483,24 @@ v_mov_b32_sdwa v1, sext(u)
 // CHECK-NEXT:{{^}}v_mov_b32_sdwa v1, sext(u)
 // CHECK-NEXT:{{^}}                        ^
 
+//==============================================================================
+// expected an identifier
+
+v_mov_b32_sdwa v5, v1 dst_sel:
+// CHECK: error: expected an identifier
+// CHECK-NEXT:{{^}}v_mov_b32_sdwa v5, v1 dst_sel:
+// CHECK-NEXT:{{^}}                              ^
+
+v_mov_b32_sdwa v5, v1 dst_sel:0
+// CHECK: error: expected an identifier
+// CHECK-NEXT:{{^}}v_mov_b32_sdwa v5, v1 dst_sel:0
+// CHECK-NEXT:{{^}}                              ^
+
+v_mov_b32_sdwa v5, v1 dst_sel:DWORD dst_unused:[UNUSED_PAD]
+// CHECK: error: expected an identifier
+// CHECK-NEXT:{{^}}v_mov_b32_sdwa v5, v1 dst_sel:DWORD dst_unused:[UNUSED_PAD]
+// CHECK-NEXT:{{^}}                                               ^
+
 //==============================================================================
 // expected an opening square bracket
 
@@ -623,6 +641,22 @@ s_waitcnt vmcnt(0) & expcnt(0) x(0)
 // CHECK-NEXT:{{^}}s_waitcnt vmcnt(0) & expcnt(0) x(0)
 // CHECK-NEXT:{{^}}                               ^
 
+//==============================================================================
+// invalid dst_sel value
+
+v_mov_b32_sdwa v5, v1 dst_sel:WORD
+// CHECK: error: invalid dst_sel value
+// CHECK-NEXT:{{^}}v_mov_b32_sdwa v5, v1 dst_sel:WORD
+// CHECK-NEXT:{{^}}                              ^
+
+//==============================================================================
+// invalid dst_unused value
+
+v_mov_b32_sdwa v5, v1 dst_unused:UNUSED
+// CHECK: error: invalid dst_unused value
+// CHECK-NEXT:{{^}}v_mov_b32_sdwa v5, v1 dst_unused:UNUSED
+// CHECK-NEXT:{{^}}                                 ^
+
 //==============================================================================
 // invalid exp target
 
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index 4329290b33ba..9188f8a3ae46 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -35,3 +35,51 @@
 
 .attribute arch, "rv32ima2p_fdc"
 # CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_f2p0_d2p0_c2p0"
+
+.attribute arch, "rv32ib"
+# CHECK: attribute      5, "rv32i2p0_b0p93_zba0p93_zbb0p93_zbc0p93_zbe0p93_zbf0p93_zbm0p93_zbp0p93_zbr0p93_zbs0p93_zbt0p93"
+
+.attribute arch, "rv32iv"
+# CHECK: attribute      5, "rv32i2p0_v0p9"
+
+.attribute arch, "rv32izba"
+# CHECK: attribute      5, "rv32i2p0_zba0p93"
+
+.attribute arch, "rv32izbb"
+# CHECK: attribute      5, "rv32i2p0_zbb0p93"
+
+.attribute arch, "rv32izbc"
+# CHECK: attribute      5, "rv32i2p0_zbc0p93"
+
+.attribute arch, "rv32izbe"
+# CHECK: attribute      5, "rv32i2p0_zbe0p93"
+
+.attribute arch, "rv32izbf"
+# CHECK: attribute      5, "rv32i2p0_zbf0p93"
+
+.attribute arch, "rv32izbm"
+# CHECK: attribute      5, "rv32i2p0_zbm0p93"
+
+.attribute arch, "rv32izbp"
+# CHECK: attribute      5, "rv32i2p0_zbp0p93"
+
+.attribute arch, "rv32izbproposedc"
+# CHECK: attribute      5, "rv32i2p0_zbproposedc0p93"
+
+.attribute arch, "rv32izbr"
+# CHECK: attribute      5, "rv32i2p0_zbr0p93"
+
+.attribute arch, "rv32izbs"
+# CHECK: attribute      5, "rv32i2p0_zbs0p93"
+
+.attribute arch, "rv32izbt"
+# CHECK: attribute      5, "rv32i2p0_zbt0p93"
+
+.attribute arch, "rv32ifzfh"
+# CHECK: attribute      5, "rv32i2p0_f2p0_zfh0p1"
+
+.attribute arch, "rv32ivzvamo_zvlsseg"
+# CHECK: attribute      5, "rv32i2p0_v0p9_zvamo0p9_zvlsseg0p9"
+
+.attribute arch, "rv32iv_zvamo0p9_zvlsseg"
+# CHECK: attribute      5, "rv32i2p0_v0p9_zvamo0p9_zvlsseg0p9"
diff --git a/llvm/test/Object/archive-extract-dir.test b/llvm/test/Object/archive-extract-dir.test
index d6a721c090b3..5182a2f91411 100644
--- a/llvm/test/Object/archive-extract-dir.test
+++ b/llvm/test/Object/archive-extract-dir.test
@@ -8,4 +8,4 @@ RUN: rm foo
 RUN: mkdir foo
 RUN: not llvm-ar x test.a foo 2>&1 | FileCheck %s
 
-CHECK: foo: {{.*}}{{[Ii]}}s a directory{{.*}}
+CHECK: foo: {{[Ii]}}s a directory
diff --git a/llvm/test/Object/archive-extract.test b/llvm/test/Object/archive-extract.test
index 589e8b8fe856..11ca12e3353b 100644
--- a/llvm/test/Object/archive-extract.test
+++ b/llvm/test/Object/archive-extract.test
@@ -58,4 +58,4 @@ RUN: llvm-ar p %p/Inputs/thin.a evenlen | FileCheck %s --check-prefix=EVENLEN
 EVENLEN: evenlen
 
 RUN: not llvm-ar p %p/Inputs/thin-path.a t/test2.o 2>&1 | FileCheck %s --check-prefix=MISSING
-MISSING: error: {{.*}}{{N|n}}o such file or directory
+MISSING: error: {{N|n}}o such file or directory
diff --git a/llvm/test/Object/directory.ll b/llvm/test/Object/directory.ll
index d9e340a587b1..3e9ac2ce202e 100644
--- a/llvm/test/Object/directory.ll
+++ b/llvm/test/Object/directory.ll
@@ -1,6 +1,6 @@
 ;RUN: rm -rf %t && mkdir -p %t
 ;RUN: not llvm-ar r %t/test.a . 2>&1 | FileCheck %s
-;CHECK: .: {{.*}}{{I|i}}s a directory{{.*}}
+;CHECK: .: {{I|i}}s a directory
 
 ;RUN: rm -f %t/test.a
 ;RUN: touch %t/a-very-long-file-name
diff --git a/llvm/test/Object/obj2yaml.test b/llvm/test/Object/obj2yaml.test
index f57af6f1fca1..f0f25ace3c1a 100644
--- a/llvm/test/Object/obj2yaml.test
+++ b/llvm/test/Object/obj2yaml.test
@@ -415,6 +415,19 @@
 # ELF-MIPSEL-NEXT:       - Offset:          0x2C
 # ELF-MIPSEL-NEXT:         Symbol:          SomeOtherFunction
 # ELF-MIPSEL-NEXT:         Type:            R_MIPS_CALL16
+# ELF-MIPSEL-NEXT:   - Type: SectionHeaderTable
+# ELF-MIPSEL-NEXT:     Sections:
+# ELF-MIPSEL-NEXT:       - Name: .text
+# ELF-MIPSEL-NEXT:       - Name: .rel.text
+# ELF-MIPSEL-NEXT:       - Name: .data
+# ELF-MIPSEL-NEXT:       - Name: .bss
+# ELF-MIPSEL-NEXT:       - Name: .mdebug.abi32
+# ELF-MIPSEL-NEXT:       - Name: .rodata.str1.1
+# ELF-MIPSEL-NEXT:       - Name: .reginfo
+# ELF-MIPSEL-NEXT:       - Name: .MIPS.abiflags
+# ELF-MIPSEL-NEXT:       - Name: .shstrtab
+# ELF-MIPSEL-NEXT:       - Name: .symtab
+# ELF-MIPSEL-NEXT:       - Name: .strtab
 # ELF-MIPSEL-NEXT: Symbols:
 # ELF-MIPSEL-NEXT:     - Name:            trivial.ll
 # ELF-MIPSEL-NEXT:       Type:            STT_FILE
@@ -460,19 +473,6 @@
 # ELF-MIPSEL-NEXT:       Binding:         STB_GLOBAL
 # ELF-MIPSEL-NEXT:     - Name:            puts
 # ELF-MIPSEL-NEXT:       Binding:         STB_GLOBAL
-# ELF-MIPSEL-NEXT: SectionHeaderTable:
-# ELF-MIPSEL-NEXT:   Sections:
-# ELF-MIPSEL-NEXT:     - Name: .text
-# ELF-MIPSEL-NEXT:     - Name: .rel.text
-# ELF-MIPSEL-NEXT:     - Name: .data
-# ELF-MIPSEL-NEXT:     - Name: .bss
-# ELF-MIPSEL-NEXT:     - Name: .mdebug.abi32
-# ELF-MIPSEL-NEXT:     - Name: .rodata.str1.1
-# ELF-MIPSEL-NEXT:     - Name: .reginfo
-# ELF-MIPSEL-NEXT:     - Name: .MIPS.abiflags
-# ELF-MIPSEL-NEXT:     - Name: .shstrtab
-# ELF-MIPSEL-NEXT:     - Name: .symtab
-# ELF-MIPSEL-NEXT:     - Name: .strtab
 # ELF-MIPSEL-NEXT: ...
 
 # RUN: obj2yaml %p/Inputs/trivial-object-test.elf-mips64el | FileCheck %s --check-prefix ELF-MIPS64EL
@@ -514,6 +514,17 @@
 # ELF-MIPS64EL-NEXT:     Relocations:
 # ELF-MIPS64EL-NEXT:       - Symbol:          zed
 # ELF-MIPS64EL-NEXT:         Type:            R_MIPS_64
+# ELF-MIPS64EL-NEXT:   - Type: SectionHeaderTable
+# ELF-MIPS64EL-NEXT:     Sections:
+# ELF-MIPS64EL-NEXT:       - Name: .text
+# ELF-MIPS64EL-NEXT:       - Name: .data
+# ELF-MIPS64EL-NEXT:       - Name: .rela.data
+# ELF-MIPS64EL-NEXT:       - Name: .bss
+# ELF-MIPS64EL-NEXT:       - Name: .MIPS.options
+# ELF-MIPS64EL-NEXT:       - Name: .pdr
+# ELF-MIPS64EL-NEXT:       - Name: .shstrtab
+# ELF-MIPS64EL-NEXT:       - Name: .symtab
+# ELF-MIPS64EL-NEXT:       - Name: .strtab
 # ELF-MIPS64EL-NEXT: Symbols:
 # ELF-MIPS64EL-NEXT:     - Name:            .text
 # ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
@@ -534,17 +545,6 @@
 # ELF-MIPS64EL-NEXT:       Section:         .pdr
 # ELF-MIPS64EL-NEXT:     - Name:            zed
 # ELF-MIPS64EL-NEXT:       Binding:         STB_GLOBAL
-# ELF-MIPS64EL-NEXT: SectionHeaderTable:
-# ELF-MIPS64EL-NEXT:   Sections:
-# ELF-MIPS64EL-NEXT:     - Name: .text
-# ELF-MIPS64EL-NEXT:     - Name: .data
-# ELF-MIPS64EL-NEXT:     - Name: .rela.data
-# ELF-MIPS64EL-NEXT:     - Name: .bss
-# ELF-MIPS64EL-NEXT:     - Name: .MIPS.options
-# ELF-MIPS64EL-NEXT:     - Name: .pdr
-# ELF-MIPS64EL-NEXT:     - Name: .shstrtab
-# ELF-MIPS64EL-NEXT:     - Name: .symtab
-# ELF-MIPS64EL-NEXT:     - Name: .strtab
 # ELF-MIPS64EL-NEXT: ...
 
 # RUN: yaml2obj %s -o %t-x86-64
diff --git a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
index 797f85d94447..97ed799f32a8 100644
--- a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
@@ -5,6 +5,8 @@ declare i8 @llvm.umin.i8(i8, i8)
 declare i8 @llvm.umax.i8(i8, i8)
 declare i8 @llvm.smin.i8(i8, i8)
 declare i8 @llvm.smax.i8(i8, i8)
+declare <3 x i8> @llvm.umin.v3i8(<3 x i8>, <3 x i8>)
+declare void @use(i8)
 
 define i8 @umin_known_bits(i8 %x, i8 %y) {
 ; CHECK-LABEL: @umin_known_bits(
@@ -45,3 +47,164 @@ define i8 @smax_known_bits(i8 %x, i8 %y) {
   %r = and i8 %m, -128
   ret i8 %r
 }
+
+define i8 @smax_sext(i5 %x, i5 %y) {
+; CHECK-LABEL: @smax_sext(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i5 @llvm.smax.i5(i5 [[X:%.*]], i5 [[Y:%.*]])
+; CHECK-NEXT:    [[M:%.*]] = sext i5 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %sx = sext i5 %x to i8
+  %sy = sext i5 %y to i8
+  %m = call i8 @llvm.smax.i8(i8 %sx, i8 %sy)
+  ret i8 %m
+}
+
+; Extra use is ok.
+
+define i8 @smin_sext(i5 %x, i5 %y) {
+; CHECK-LABEL: @smin_sext(
+; CHECK-NEXT:    [[SY:%.*]] = sext i5 [[Y:%.*]] to i8
+; CHECK-NEXT:    call void @use(i8 [[SY]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i5 @llvm.smin.i5(i5 [[X:%.*]], i5 [[Y]])
+; CHECK-NEXT:    [[M:%.*]] = sext i5 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %sx = sext i5 %x to i8
+  %sy = sext i5 %y to i8
+  call void @use(i8 %sy)
+  %m = call i8 @llvm.smin.i8(i8 %sx, i8 %sy)
+  ret i8 %m
+}
+
+; Sext doesn't change unsigned min/max comparison of narrow values.
+
+define i8 @umax_sext(i5 %x, i5 %y) {
+; CHECK-LABEL: @umax_sext(
+; CHECK-NEXT:    [[SX:%.*]] = sext i5 [[X:%.*]] to i8
+; CHECK-NEXT:    call void @use(i8 [[SX]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i5 @llvm.umax.i5(i5 [[X]], i5 [[Y:%.*]])
+; CHECK-NEXT:    [[M:%.*]] = sext i5 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %sx = sext i5 %x to i8
+  call void @use(i8 %sx)
+  %sy = sext i5 %y to i8
+  %m = call i8 @llvm.umax.i8(i8 %sx, i8 %sy)
+  ret i8 %m
+}
+
+define <3 x i8> @umin_sext(<3 x i5> %x, <3 x i5> %y) {
+; CHECK-LABEL: @umin_sext(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <3 x i5> @llvm.umin.v3i5(<3 x i5> [[X:%.*]], <3 x i5> [[Y:%.*]])
+; CHECK-NEXT:    [[M:%.*]] = sext <3 x i5> [[TMP1]] to <3 x i8>
+; CHECK-NEXT:    ret <3 x i8> [[M]]
+;
+  %sx = sext <3 x i5> %x to <3 x i8>
+  %sy = sext <3 x i5> %y to <3 x i8>
+  %m = call <3 x i8> @llvm.umin.v3i8(<3 x i8> %sx, <3 x i8> %sy)
+  ret <3 x i8> %m
+}
+
+; Negative test - zext may change sign of inputs
+
+define i8 @smax_zext(i5 %x, i5 %y) {
+; CHECK-LABEL: @smax_zext(
+; CHECK-NEXT:    [[ZX:%.*]] = zext i5 [[X:%.*]] to i8
+; CHECK-NEXT:    [[ZY:%.*]] = zext i5 [[Y:%.*]] to i8
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smax.i8(i8 [[ZX]], i8 [[ZY]])
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %zx = zext i5 %x to i8
+  %zy = zext i5 %y to i8
+  %m = call i8 @llvm.smax.i8(i8 %zx, i8 %zy)
+  ret i8 %m
+}
+
+; Negative test - zext may change sign of inputs
+
+define i8 @smin_zext(i5 %x, i5 %y) {
+; CHECK-LABEL: @smin_zext(
+; CHECK-NEXT:    [[ZX:%.*]] = zext i5 [[X:%.*]] to i8
+; CHECK-NEXT:    [[ZY:%.*]] = zext i5 [[Y:%.*]] to i8
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.smin.i8(i8 [[ZX]], i8 [[ZY]])
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %zx = zext i5 %x to i8
+  %zy = zext i5 %y to i8
+  %m = call i8 @llvm.smin.i8(i8 %zx, i8 %zy)
+  ret i8 %m
+}
+
+define i8 @umax_zext(i5 %x, i5 %y) {
+; CHECK-LABEL: @umax_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i5 @llvm.umax.i5(i5 [[X:%.*]], i5 [[Y:%.*]])
+; CHECK-NEXT:    [[M:%.*]] = zext i5 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %zx = zext i5 %x to i8
+  %zy = zext i5 %y to i8
+  %m = call i8 @llvm.umax.i8(i8 %zx, i8 %zy)
+  ret i8 %m
+}
+
+define i8 @umin_zext(i5 %x, i5 %y) {
+; CHECK-LABEL: @umin_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i5 @llvm.umin.i5(i5 [[X:%.*]], i5 [[Y:%.*]])
+; CHECK-NEXT:    [[M:%.*]] = zext i5 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %zx = zext i5 %x to i8
+  %zy = zext i5 %y to i8
+  %m = call i8 @llvm.umin.i8(i8 %zx, i8 %zy)
+  ret i8 %m
+}
+
+; Negative test - mismatched types
+
+define i8 @umin_zext_types(i6 %x, i5 %y) {
+; CHECK-LABEL: @umin_zext_types(
+; CHECK-NEXT:    [[ZX:%.*]] = zext i6 [[X:%.*]] to i8
+; CHECK-NEXT:    [[ZY:%.*]] = zext i5 [[Y:%.*]] to i8
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[ZX]], i8 [[ZY]])
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %zx = zext i6 %x to i8
+  %zy = zext i5 %y to i8
+  %m = call i8 @llvm.umin.i8(i8 %zx, i8 %zy)
+  ret i8 %m
+}
+
+; Negative test - mismatched extends
+
+define i8 @umin_ext(i5 %x, i5 %y) {
+; CHECK-LABEL: @umin_ext(
+; CHECK-NEXT:    [[SX:%.*]] = sext i5 [[X:%.*]] to i8
+; CHECK-NEXT:    [[ZY:%.*]] = zext i5 [[Y:%.*]] to i8
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[SX]], i8 [[ZY]])
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %sx = sext i5 %x to i8
+  %zy = zext i5 %y to i8
+  %m = call i8 @llvm.umin.i8(i8 %sx, i8 %zy)
+  ret i8 %m
+}
+
+; Negative test - too many uses.
+
+define i8 @umin_zext_uses(i5 %x, i5 %y) {
+; CHECK-LABEL: @umin_zext_uses(
+; CHECK-NEXT:    [[ZX:%.*]] = zext i5 [[X:%.*]] to i8
+; CHECK-NEXT:    call void @use(i8 [[ZX]])
+; CHECK-NEXT:    [[ZY:%.*]] = zext i5 [[Y:%.*]] to i8
+; CHECK-NEXT:    call void @use(i8 [[ZY]])
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umin.i8(i8 [[ZX]], i8 [[ZY]])
+; CHECK-NEXT:    ret i8 [[M]]
+;
+  %zx = zext i5 %x to i8
+  call void @use(i8 %zx)
+  %zy = zext i5 %y to i8
+  call void @use(i8 %zy)
+  %m = call i8 @llvm.umin.i8(i8 %zx, i8 %zy)
+  ret i8 %m
+}
diff --git a/llvm/test/tools/dsymutil/X86/papertrail-warnings.test b/llvm/test/tools/dsymutil/X86/papertrail-warnings.test
index 6f0a5e11a7c1..1954994c9e1e 100644
--- a/llvm/test/tools/dsymutil/X86/papertrail-warnings.test
+++ b/llvm/test/tools/dsymutil/X86/papertrail-warnings.test
@@ -8,7 +8,7 @@ CHECK:    DW_AT_name {{.*}}"/Inputs/basic1.macho.x86_64.o"
 CHECK:    DW_TAG_constant [2]
 CHECK:      DW_AT_name {{.*}}"dsymutil_warning"
 CHECK:      DW_AT_artificial [DW_FORM_flag]	(0x01)
-CHECK:      DW_AT_const_value {{.*}}"unable to open object file: {{.*}}{{[Nn]}}o such file or directory"
+CHECK:      DW_AT_const_value {{.*}}"unable to open object file: {{[Nn]}}o such file or directory"
 CHECK:    NULL
 CHECK:  Compile Unit:
 CHECK:  DW_TAG_compile_unit [1] *
@@ -17,7 +17,7 @@ CHECK:    DW_AT_name {{.*}}"/Inputs/basic2.macho.x86_64.o"
 CHECK:    DW_TAG_constant [2]
 CHECK:      DW_AT_name {{.*}}"dsymutil_warning"
 CHECK:      DW_AT_artificial [DW_FORM_flag]	(0x01)
-CHECK:      DW_AT_const_value {{.*}}"unable to open object file: {{.*}}{{[Nn]}}o such file or directory"
+CHECK:      DW_AT_const_value {{.*}}"unable to open object file: {{[Nn]}}o such file or directory"
 CHECK:    NULL
 CHECK:  Compile Unit:
 CHECK:  DW_TAG_compile_unit [1] *
@@ -26,5 +26,5 @@ CHECK:    DW_AT_name {{.*}}"/Inputs/basic3.macho.x86_64.o"
 CHECK:    DW_TAG_constant [2]
 CHECK:      DW_AT_name {{.*}}"dsymutil_warning"
 CHECK:      DW_AT_artificial [DW_FORM_flag]	(0x01)
-CHECK:      DW_AT_const_value {{.*}}"unable to open object file: {{.*}}{{[Nn]}}o such file or directory"
+CHECK:      DW_AT_const_value {{.*}}"unable to open object file: {{[Nn]}}o such file or directory"
 CHECK:    NULL
diff --git a/llvm/test/tools/llvm-ar/error-opening-directory.test b/llvm/test/tools/llvm-ar/error-opening-directory.test
index 5a6491bd29c8..c3068871230f 100644
--- a/llvm/test/tools/llvm-ar/error-opening-directory.test
+++ b/llvm/test/tools/llvm-ar/error-opening-directory.test
@@ -9,4 +9,4 @@
 # RUN: not llvm-ar p %t/tmpDir 2>&1 | \
 # RUN:    FileCheck %s --check-prefix=IS-DIR -DARCHIVE=%t/tmpDir
 
-# IS-DIR: error: unable to open '[[ARCHIVE]]': {{.*}}{{[iI]}}s a directory
+# IS-DIR: error: unable to open '[[ARCHIVE]]': {{[iI]}}s a directory
diff --git a/llvm/test/tools/llvm-ar/missing-thin-archive-member.test b/llvm/test/tools/llvm-ar/missing-thin-archive-member.test
index 7dd9f26a40ba..c1d67cdce2f1 100644
--- a/llvm/test/tools/llvm-ar/missing-thin-archive-member.test
+++ b/llvm/test/tools/llvm-ar/missing-thin-archive-member.test
@@ -14,4 +14,4 @@
 # RUN: rm %t-temp.txt
 # RUN: not llvm-ar p %t-archive.a 2>&1 | FileCheck %s --check-prefix=DELETED
 
-# DELETED: error: {{.*}}{{[Nn]}}o such file or directory
+# DELETED: error: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-ar/move.test b/llvm/test/tools/llvm-ar/move.test
index 285d7d751eef..e62b8ccd609d 100644
--- a/llvm/test/tools/llvm-ar/move.test
+++ b/llvm/test/tools/llvm-ar/move.test
@@ -84,7 +84,7 @@
 # RUN: not llvm-ar m %t/missing.a %t/missing.txt 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-FILE -DFILE=%t/missing.txt
 
-# MISSING-FILE: error: [[FILE]]: {{.*}}{{[nN]}}o such file or directory
+# MISSING-FILE: error: [[FILE]]: {{[nN]}}o such file or directory
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-ar/print.test b/llvm/test/tools/llvm-ar/print.test
index b05a6e5cb200..247a3d44022f 100644
--- a/llvm/test/tools/llvm-ar/print.test
+++ b/llvm/test/tools/llvm-ar/print.test
@@ -75,7 +75,7 @@
 # RUN: not llvm-ar p %t/missing.a 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-ARCHIVE -DARCHIVE=%t/missing.a
 
-# MISSING-ARCHIVE: error: unable to load '[[ARCHIVE]]': {{.*}}{{[nN]}}o such file or directory
+# MISSING-ARCHIVE: error: unable to load '[[ARCHIVE]]': {{[nN]}}o such file or directory
 
 ## Member does not exist:
 # RUN: not llvm-ar p %t/archive.a %t-missing.txt 2>&1 \
diff --git a/llvm/test/tools/llvm-ar/quick-append.test b/llvm/test/tools/llvm-ar/quick-append.test
index c5f7a4d755d2..968460ec88fa 100644
--- a/llvm/test/tools/llvm-ar/quick-append.test
+++ b/llvm/test/tools/llvm-ar/quick-append.test
@@ -60,7 +60,7 @@
 # RUN: not llvm-ar qc %t/missing.a %t/missing.txt 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-FILE -DFILE=%t/missing.txt
 
-# MISSING-FILE: error: [[FILE]]: {{.*}}{{[nN]}}o such file or directory
+# MISSING-FILE: error: [[FILE]]: {{[nN]}}o such file or directory
 
 ## Create and append members to a thin archive:
 # RUN: llvm-ar qcT %t/thin-multiple.a %t/1.o
diff --git a/llvm/test/tools/llvm-ar/replace.test b/llvm/test/tools/llvm-ar/replace.test
index ce6658d3f934..389ac60270c4 100644
--- a/llvm/test/tools/llvm-ar/replace.test
+++ b/llvm/test/tools/llvm-ar/replace.test
@@ -91,7 +91,7 @@
 # RUN: not llvm-ar r %t/missing.a %t/missing.txt 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-FILE -DFILE=%t/missing.txt
 
-# MISSING-FILE: error: [[FILE]]: {{.*}}{{[Nn]}}o such file or directory
+# MISSING-FILE: error: [[FILE]]: {{[Nn]}}o such file or directory
 
 ## Create and Replace member of thin archive:
 # RUN: llvm-ar rcT %t/thin.a %t/1.o %t/2.o %t/3.o
diff --git a/llvm/test/tools/llvm-ar/response.test b/llvm/test/tools/llvm-ar/response.test
index bdb6ae03d129..d9262481172a 100644
--- a/llvm/test/tools/llvm-ar/response.test
+++ b/llvm/test/tools/llvm-ar/response.test
@@ -34,8 +34,8 @@
 # RUN: echo -e 'rc %/t.a blah\\foo' > %t-rsp.txt
 # RUN: not llvm-ar --rsp-quoting=windows @%t-rsp.txt 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=WIN
-# WIN: error: blah\foo: {{.*}}{{[Nn]}}o such file or directory
+# WIN: error: blah\foo: {{[Nn]}}o such file or directory
 
 # RUN: not llvm-ar -rsp-quoting posix @%t-rsp.txt 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=POSIX
-# POSIX: error: blahfoo: {{.*}}{{[Nn]}}o such file or directory
+# POSIX: error: blahfoo: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-cxxdump/trivial.test b/llvm/test/tools/llvm-cxxdump/trivial.test
index 6a2582692520..8df0bc32e1e3 100644
--- a/llvm/test/tools/llvm-cxxdump/trivial.test
+++ b/llvm/test/tools/llvm-cxxdump/trivial.test
@@ -63,4 +63,4 @@ ELF-I386-NEXT: _ZTV1A[8]: _ZN1A1fEv
 MIXEDARCOFF-I386:      ??_7S@@6B@[0]: ??_R4S@@6B@
 
 RUN: not llvm-cxxdump %t.blah 2>&1 | FileCheck --check-prefix=ENOENT %s
-ENOENT: {{.*}}.blah: error: {{.*}}{{[Nn]}}o such file or directory
+ENOENT: {{.*}}.blah: error: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-dwp/WebAssembly/lit.local.cfg b/llvm/test/tools/llvm-dwp/WebAssembly/lit.local.cfg
new file mode 100644
index 000000000000..5ea762c84049
--- /dev/null
+++ b/llvm/test/tools/llvm-dwp/WebAssembly/lit.local.cfg
@@ -0,0 +1,4 @@
+if not 'WebAssembly' in config.root.targets:
+    config.unsupported = True
+
+config.suffixes = ['.s']
diff --git a/llvm/test/tools/llvm-dwp/WebAssembly/simple_dwo.s b/llvm/test/tools/llvm-dwp/WebAssembly/simple_dwo.s
new file mode 100644
index 000000000000..f3e88c97d581
--- /dev/null
+++ b/llvm/test/tools/llvm-dwp/WebAssembly/simple_dwo.s
@@ -0,0 +1,11 @@
+# RUN: llvm-mc %s -filetype obj -triple wasm32-unknown-unknown -o %t.o \
+# RUN:         -split-dwarf-file=%t.dwo -dwarf-version=5
+# RUN: llvm-dwp %t.dwo -o %t.dwp
+# RUN: llvm-dwarfdump -v %t.dwp | FileCheck %s
+
+# This test checks whether llvm-dwp is able to emit object files of the same
+# triple as its inputs.
+
+# CHECK: file format WASM
+
+# Empty file, we just care about the file type.
diff --git a/llvm/test/tools/llvm-libtool-darwin/filelist.test b/llvm/test/tools/llvm-libtool-darwin/filelist.test
index a8d8dac95a49..bb606f0585fa 100644
--- a/llvm/test/tools/llvm-libtool-darwin/filelist.test
+++ b/llvm/test/tools/llvm-libtool-darwin/filelist.test
@@ -65,13 +65,13 @@
 # RUN: not llvm-libtool-darwin -static -o %t.lib -filelist %t.invalid-list.txt 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=FILE-ERROR -DFILE=no-such-file
 
-# FILE-ERROR: error: '[[FILE]]': {{.*}}{{[nN]}}o such file or directory
+# FILE-ERROR: error: '[[FILE]]': {{[nN]}}o such file or directory
 
 ## Check that an error is thrown when the directory exists but does not contain the requested file:
 # RUN: not llvm-libtool-darwin -static -o %t.lib -filelist %t.invalid-list.txt,%t/dirname 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=DIR-ERROR -DDIR=%t/dirname -DFILE=no-such-file
 
-# DIR-ERROR: error: '[[DIR]]{{[/\\]}}[[FILE]]': {{.*}}{{[nN]}}o such file or directory
+# DIR-ERROR: error: '[[DIR]]{{[/\\]}}[[FILE]]': {{[nN]}}o such file or directory
 
 ## Check that an error is thrown when a file is in the cwd but dirname is specified:
 # RUN: yaml2obj %S/Inputs/input2.yaml -o %basename_t.tmp-input2.o
diff --git a/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test b/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test
index 8ffe2ef16993..da1bcd4e9625 100644
--- a/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test
+++ b/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test
@@ -28,7 +28,7 @@
 # RUN: not llvm-libtool-darwin -static -o %t.lib %t.missing 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=NO-FILE -DFILE=%t.missing
 
-# NO-FILE: error: '[[FILE]]': {{.*}}{{[nN]}}o such file or directory
+# NO-FILE: error: '[[FILE]]': {{[nN]}}o such file or directory
 
 ## Input file is not an object file:
 # RUN: touch %t.invalid
diff --git a/llvm/test/tools/llvm-lipo/create-arch.test b/llvm/test/tools/llvm-lipo/create-arch.test
index 8aa5358621dc..08e9e18ca5e3 100644
--- a/llvm/test/tools/llvm-lipo/create-arch.test
+++ b/llvm/test/tools/llvm-lipo/create-arch.test
@@ -14,4 +14,4 @@
 # INVALID_ARCH: error: Invalid architecture: i3866
 #
 # RUN: not llvm-lipo -arch i386 %t-33.o -create -o /dev/null 2>&1 | FileCheck --check-prefix=INVALID_FILE %s
-# INVALID_FILE: {{.*}}{{[nN]}}o such file or directory
+# INVALID_FILE: {{[nN]}}o such file or directory
diff --git a/llvm/test/tools/llvm-lipo/replace-invalid-input.test b/llvm/test/tools/llvm-lipo/replace-invalid-input.test
index e43630008dfb..594594bc2ee6 100644
--- a/llvm/test/tools/llvm-lipo/replace-invalid-input.test
+++ b/llvm/test/tools/llvm-lipo/replace-invalid-input.test
@@ -12,7 +12,7 @@
 # INPUT_ARGS: error: replace expects a single input file
 
 # RUN: not llvm-lipo %t-universal.o -replace i386 %t-33.o  -o %t.o 2>&1 | FileCheck --check-prefix=INVALID_FILE %s
-# INVALID_FILE: {{.*}}{{[nN]}}o such file or directory
+# INVALID_FILE: {{[nN]}}o such file or directory
 
 # RUN: not llvm-lipo %t-universal.o -replace i3866 %t-32.o -o %t.o 2>&1 | FileCheck --check-prefix=INVALID_ARCH %s
 # INVALID_ARCH: error: Invalid architecture: i3866
diff --git a/llvm/test/tools/llvm-lto/error.ll b/llvm/test/tools/llvm-lto/error.ll
index d443770fe05e..e1f9e8813ab3 100644
--- a/llvm/test/tools/llvm-lto/error.ll
+++ b/llvm/test/tools/llvm-lto/error.ll
@@ -1,5 +1,5 @@
 ; RUN: not llvm-lto foobar 2>&1 | FileCheck %s
-; CHECK: llvm-lto: error loading file 'foobar': {{.*}}{{N|n}}o such file or directory
+; CHECK: llvm-lto: error loading file 'foobar': {{N|n}}o such file or directory
 
 ; RUN: not llvm-lto --list-symbols-only %S/Inputs/empty.bc 2>&1 | FileCheck %s --check-prefix=CHECK-LIST
 ; CHECK-LIST: llvm-lto: error loading file '{{.*}}/Inputs/empty.bc': The file was not recognized as a valid object file
diff --git a/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll b/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
index c76f3f0a8052..948c52202204 100644
--- a/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
+++ b/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
@@ -22,4 +22,4 @@ define void @patatino() {
 ; Try to save statistics to an invalid file.
 ; RUN: not llvm-lto2 run %t1.bc -o %t.o -r %t1.bc,patatino,px \
 ; RUN:     -stats-file=%t2/foo.stats 2>&1 | FileCheck --check-prefix=ERROR %s
-; ERROR: LTO::run failed: {{.*}}{{[Nn]}}o such file or directory
+; ERROR: LTO::run failed: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-mc/basic.test b/llvm/test/tools/llvm-mc/basic.test
index 695329c3e61d..c7480dc6a87f 100644
--- a/llvm/test/tools/llvm-mc/basic.test
+++ b/llvm/test/tools/llvm-mc/basic.test
@@ -1,3 +1,3 @@
 # RUN: not llvm-mc %t.blah -o /dev/null 2>&1 | FileCheck --check-prefix=ENOENT %s
 
-# ENOENT: {{.*}}.blah: {{.*}}{{[Nn]}}o such file or directory
+# ENOENT: {{.*}}.blah: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-mca/invalid_input_file_name.test b/llvm/test/tools/llvm-mca/invalid_input_file_name.test
index 5bc671932494..27fa972f29a4 100644
--- a/llvm/test/tools/llvm-mca/invalid_input_file_name.test
+++ b/llvm/test/tools/llvm-mca/invalid_input_file_name.test
@@ -1,3 +1,3 @@
 # RUN: not llvm-mca %t.blah -o /dev/null 2>&1 | FileCheck --check-prefix=ENOENT %s
 
-# ENOENT: {{.*}}.blah: {{.*}}{{[Nn]}}o such file or directory
+# ENOENT: {{.*}}.blah: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-ml/basic.test b/llvm/test/tools/llvm-ml/basic.test
index f9df410bde75..960eb47f6cc5 100644
--- a/llvm/test/tools/llvm-ml/basic.test
+++ b/llvm/test/tools/llvm-ml/basic.test
@@ -1,3 +1,3 @@
 # RUN: not llvm-ml %t.blah.asm /Fo /dev/null 2>&1 | FileCheck --check-prefix=ENOENT %s
 
-# ENOENT: {{.*}}.blah.asm: {{.*}}{{[Nn]}}o such file or directory
+# ENOENT: {{.*}}.blah.asm: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-objcopy/COFF/add-section.test b/llvm/test/tools/llvm-objcopy/COFF/add-section.test
index 464c6d7c1b43..e4938db9135f 100644
--- a/llvm/test/tools/llvm-objcopy/COFF/add-section.test
+++ b/llvm/test/tools/llvm-objcopy/COFF/add-section.test
@@ -61,7 +61,7 @@
 ## to be added does not exist.
 # RUN: not llvm-objcopy --add-section=.another.section=%t2 %t %t3 2>&1 | FileCheck -DFILE1=%t -DFILE2=%t2 %s --check-prefixes=ERR1
 
-# ERR1: error: '[[FILE1]]': '[[FILE2]]': {{.*}}{{[Nn]}}o such file or directory
+# ERR1: error: '[[FILE1]]': '[[FILE2]]': {{[Nn]}}o such file or directory
 
 ## Another negative test for invalid --add-sections command line argument.
 # RUN: not llvm-objcopy --add-section=.another.section %t %t3 2>&1 | FileCheck %s --check-prefixes=ERR2
diff --git a/llvm/test/tools/llvm-objcopy/ELF/add-section.test b/llvm/test/tools/llvm-objcopy/ELF/add-section.test
index a9de79b87bc3..290761f82bc1 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/add-section.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/add-section.test
@@ -39,7 +39,7 @@ Sections:
 ## to be added does not exist.
 # RUN: not llvm-objcopy --add-section=.section.name=%t.missing %t %t.out 2>&1 | FileCheck -DFILE1=%t -DFILE2=%t.missing %s --check-prefixes=ERR1
 
-# ERR1: error: '[[FILE1]]': '[[FILE2]]': {{.*}}{{[Nn]}}o such file or directory
+# ERR1: error: '[[FILE1]]': '[[FILE2]]': {{[Nn]}}o such file or directory
 
 ## Negative test for invalid --add-sections argument - missing '='.
 # RUN: not llvm-objcopy --add-section=.section.name %t %t.out 2>&1 | FileCheck %s --check-prefixes=ERR2
diff --git a/llvm/test/tools/llvm-objcopy/ELF/error-format.test b/llvm/test/tools/llvm-objcopy/ELF/error-format.test
index 2dcfd9ee0b35..8a74fb0d1340 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/error-format.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/error-format.test
@@ -3,7 +3,7 @@
 # RUN: not llvm-objcopy %S/non-existent 2>&1 | FileCheck --check-prefix=ERR %s -DTOOL=objcopy
 # RUN: not llvm-strip %S/non-existent 2>&1 | FileCheck --check-prefix=ERR %s -DTOOL=strip
 
-# ERR: llvm-[[TOOL]]{{(\.exe)?}}: error: '{{.*}}': {{.*}}{{[Nn]}}o such file or directory
+# ERR: llvm-[[TOOL]]{{(\.exe)?}}: error: '{{.*}}': {{[Nn]}}o such file or directory
 
 ## Currently llvm-objcopy does not issue warnings, so it is not tested.
 
diff --git a/llvm/test/tools/llvm-objcopy/MachO/add-section-error.test b/llvm/test/tools/llvm-objcopy/MachO/add-section-error.test
index 62422adc4e7c..a6847ec878bf 100644
--- a/llvm/test/tools/llvm-objcopy/MachO/add-section-error.test
+++ b/llvm/test/tools/llvm-objcopy/MachO/add-section-error.test
@@ -6,7 +6,7 @@
 ## Error case 1: Nonexistent input file is specified by --add-section.
 # RUN: not llvm-objcopy --add-section __TEXT,__text=%t.missing %t %t.nonexistent-file 2>&1 \
 # RUN:   | FileCheck %s -DINPUT=%t -DSECTION_DATA_FILE=%t.missing --check-prefix=NONEXSITENT-FILE
-# NONEXSITENT-FILE: error: '[[INPUT]]': '[[SECTION_DATA_FILE]]': {{.*}}{{[Nn]}}o such file or directory
+# NONEXSITENT-FILE: error: '[[INPUT]]': '[[SECTION_DATA_FILE]]': {{[Nn]}}o such file or directory
 
 ## Error case 2: Too long segment name.
 # RUN: not llvm-objcopy --add-section __TOOOOOOOOO_LONG,__text=%t.data %t %t.too-long-seg-name 2>&1 \
diff --git a/llvm/test/tools/llvm-objcopy/redefine-symbols.test b/llvm/test/tools/llvm-objcopy/redefine-symbols.test
index f06427212f28..8f6cba0c8243 100644
--- a/llvm/test/tools/llvm-objcopy/redefine-symbols.test
+++ b/llvm/test/tools/llvm-objcopy/redefine-symbols.test
@@ -14,7 +14,7 @@
 # MISSING-SYM-NAME: error: {{.*}}.rename.txt:2: missing new symbol name
 
 # RUN: not llvm-objcopy --redefine-syms %t.rename-none.txt %t /dev/null 2>&1 | FileCheck %s --check-prefix=NO-FILE
-# NO-FILE: error: '{{.*}}.rename-none.txt': {{.*}}{{[Nn]}}o such file or directory
+# NO-FILE: error: '{{.*}}.rename-none.txt': {{[Nn]}}o such file or directory
 
 !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-objcopy/wasm/dump-section.test b/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
index d87e349f30a8..19620f7a3d33 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
@@ -11,7 +11,7 @@
 # CHECK: 0000020 63 6c 61 6e 67 05 39 2e 30 2e 30
 
 # NONEXISTENT: section 'nonexistent' not found
-# DIROUT: error: {{.*}}/bar': {{.*}}{{[nN]}}o such file or directory
+# DIROUT: error: {{.*}}/bar': {{[nN]}}o such file or directory
 
 ## Check that dumping and removing a section works in the same invocation
 # RUN: llvm-objcopy --dump-section=producers=%t.sec --remove-section=producers %t %t2
diff --git a/llvm/test/tools/llvm-profdata/weight-instr.test b/llvm/test/tools/llvm-profdata/weight-instr.test
index d35997e10439..83ea1ca6f6fa 100644
--- a/llvm/test/tools/llvm-profdata/weight-instr.test
+++ b/llvm/test/tools/llvm-profdata/weight-instr.test
@@ -74,7 +74,7 @@ INVALID_WEIGHT: error: Input weight must be a positive integer.
 
 4- Bad merge: input path does not exist
 RUN: not llvm-profdata merge -instr -weighted-input=3,%p/Inputs/does-not-exist.profdata -weighted-input=2,%p/Inputs/does-not-exist-either.profdata -o %t.out 2>&1 | FileCheck %s -check-prefix=INVALID_INPUT
-INVALID_INPUT: {{.*}}: {{.*}}does-not-exist.profdata: {{.*}}{{[Nn]}}o such file or directory
+INVALID_INPUT: {{.*}}: {{.*}}does-not-exist.profdata: {{[Nn]}}o such file or directory
 
 5- No inputs
 RUN: not llvm-profdata merge -instr -o %t.out 2>&1 | FileCheck %s -check-prefix=NO_INPUT
diff --git a/llvm/test/tools/llvm-profdata/weight-sample.test b/llvm/test/tools/llvm-profdata/weight-sample.test
index cd9ecdbec829..635e6212e790 100644
--- a/llvm/test/tools/llvm-profdata/weight-sample.test
+++ b/llvm/test/tools/llvm-profdata/weight-sample.test
@@ -49,7 +49,7 @@ INVALID_WEIGHT: error: Input weight must be a positive integer.
 
 4- Bad merge: input path does not exist
 RUN: not llvm-profdata merge -sample -weighted-input=3,%p/Inputs/does-not-exist.proftext -weighted-input=2,%p/Inputs/does-not-exist-either.proftext -o %t.out 2>&1 | FileCheck %s -check-prefix=INVALID_INPUT
-INVALID_INPUT: {{.*}}: {{.*}}does-not-exist.proftext: {{.*}}{{[Nn]}}o such file or directory
+INVALID_INPUT: {{.*}}: {{.*}}does-not-exist.proftext: {{[Nn]}}o such file or directory
 
 5- No inputs
 RUN: not llvm-profdata merge -sample -o %t.out 2>&1 | FileCheck %s -check-prefix=NO_INPUT
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
index cdf205981420..1a1c6dd4d0d1 100644
--- a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
+++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
@@ -447,6 +447,8 @@ Sections:
     Flags:   [ SHF_ALLOC ]
     Address: 0x300
     Offset:  0x300
+  - Type: SectionHeaderTable
+    NoHeaders: true
 DynamicSymbols:
   - Name: foo
 ProgramHeaders:
@@ -457,5 +459,3 @@ ProgramHeaders:
   - Type:     PT_DYNAMIC
     FirstSec: .dynamic
     LastSec:  .dynamic
-SectionHeaderTable:
-  NoHeaders: true
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test
index ae99d76b8f8b..4702af6af694 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc-no-section-headers.test
@@ -50,6 +50,8 @@ Sections:
         Value: 0x0
   - Name:    .dynsym
     Type:    SHT_DYNSYM
+  - Type: SectionHeaderTable
+    NoHeaders: true
 DynamicSymbols:
   - Name: foo
 ProgramHeaders:
@@ -59,5 +61,3 @@ ProgramHeaders:
   - Type:     PT_DYNAMIC
     FirstSec: .dynamic
     LastSec:  .dynamic
-SectionHeaderTable:
-  NoHeaders: true
diff --git a/llvm/test/tools/llvm-readobj/ELF/file-headers.test b/llvm/test/tools/llvm-readobj/ELF/file-headers.test
index 4b78007c0f4e..97ab9f092b22 100644
--- a/llvm/test/tools/llvm-readobj/ELF/file-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/file-headers.test
@@ -215,8 +215,9 @@ FileHeader:
 ## The index of the section name string table is too large.
 ## The section would be past the EOF.
   EShStrNdx: [[SHSTRNDX=0x3000]]
-SectionHeaderTable:
-  NoHeaders: true
+Sections:
+  - Type: SectionHeaderTable
+    NoHeaders: true
 
 ## Check we don't dump anything except the file header when the section header table can't be read.
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-table.test b/llvm/test/tools/llvm-readobj/ELF/hash-table.test
index ad52a98639be..db68ebdbcc6e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-table.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-table.test
@@ -113,8 +113,8 @@ Sections:
         Value: 0x0
       - Tag:   DT_NULL
         Value: 0x0
-SectionHeaderTable:
-  NoHeaders: [[NOHEADERS=false]]
+  - Type: SectionHeaderTable
+    NoHeaders: [[NOHEADERS=false]]
 ProgramHeaders:
   - Type:     PT_LOAD
     FirstSec: .hash
diff --git a/llvm/test/tools/llvm-readobj/ELF/malformed-pt-dynamic.test b/llvm/test/tools/llvm-readobj/ELF/malformed-pt-dynamic.test
index 02df7117a832..0666674b43d5 100644
--- a/llvm/test/tools/llvm-readobj/ELF/malformed-pt-dynamic.test
+++ b/llvm/test/tools/llvm-readobj/ELF/malformed-pt-dynamic.test
@@ -137,10 +137,10 @@ Sections:
     Entries:
       - Tag:   DT_NULL
         Value: 0
+  - Type: SectionHeaderTable
+    NoHeaders: [[NOHEADERS=false]]
 ProgramHeaders:
   - Type:     PT_DYNAMIC
     FileSize: [[FILESIZE=<none>]]
     FirstSec: .dynamic
     LastSec:  .dynamic
-SectionHeaderTable:
-  NoHeaders: [[NOHEADERS=false]]
diff --git a/llvm/test/tools/llvm-readobj/ELF/symtab-shndx.test b/llvm/test/tools/llvm-readobj/ELF/symtab-shndx.test
index 7ee9c2c0296a..493222adc302 100644
--- a/llvm/test/tools/llvm-readobj/ELF/symtab-shndx.test
+++ b/llvm/test/tools/llvm-readobj/ELF/symtab-shndx.test
@@ -313,6 +313,8 @@ Sections:
     Entries: [ 0, 1 ]
     Offset:  0x2000
     Address: 0x2000
+  - Type:      SectionHeaderTable
+    NoHeaders: true
 DynamicSymbols:
   - Name:  dynsym1
     Index: SHN_XINDEX
@@ -327,5 +329,3 @@ ProgramHeaders:
     VAddr:    0x1000
     FirstSec: .dynamic
     LastSec:  .dynamic
-SectionHeaderTable:
-  NoHeaders: true
diff --git a/llvm/test/tools/llvm-readobj/ELF/thin-archive-paths.test b/llvm/test/tools/llvm-readobj/ELF/thin-archive-paths.test
index 0a8eca83ccf1..996fae42cedb 100644
--- a/llvm/test/tools/llvm-readobj/ELF/thin-archive-paths.test
+++ b/llvm/test/tools/llvm-readobj/ELF/thin-archive-paths.test
@@ -16,7 +16,7 @@
 # RUN: rm a/b/1.o
 # RUN: not llvm-readobj --file-headers a/relative.a 2>&1 | FileCheck %s --check-prefix=ERR1
 # RUN: not llvm-readelf --file-headers a/relative.a 2>&1 | FileCheck %s --check-prefix=ERR1
-# ERR1: error: 'a/relative.a': 'b/1.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR1: error: 'a/relative.a': 'b/1.o': {{[Nn]}}o such file or directory
 
 # RUN: mkdir -p %t/c
 # RUN: yaml2obj %s -o a/b/1.o
@@ -30,7 +30,7 @@
 # RUN: rm a/b/1.o
 # RUN: not llvm-readobj --file-headers %/t/c/absolute.a 2>&1 | FileCheck %s --check-prefix=ERR2 -DDIR=%/t
 # RUN: not llvm-readelf --file-headers %/t/c/absolute.a 2>&1 | FileCheck %s --check-prefix=ERR2 -DDIR=%/t
-# ERR2: error: '[[DIR]]/c/absolute.a': '[[DIR]]/a/b/1.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR2: error: '[[DIR]]/c/absolute.a': '[[DIR]]/a/b/1.o': {{[Nn]}}o such file or directory
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/basic.test b/llvm/test/tools/llvm-readobj/basic.test
index db6de66b7bf5..ddf6acb5c44d 100644
--- a/llvm/test/tools/llvm-readobj/basic.test
+++ b/llvm/test/tools/llvm-readobj/basic.test
@@ -2,7 +2,7 @@
 RUN: not llvm-readobj %t.blah 2>&1 | FileCheck --check-prefix=ENOENT -DTOOL=readobj %s
 RUN: not llvm-readelf %t.blah 2>&1 | FileCheck --check-prefix=ENOENT -DTOOL=readelf %s
 
-ENOENT: llvm-[[TOOL]]{{(\.exe)?}}: error: '{{.*}}.blah': {{.*}}{{[Nn]}}o such file or directory
+ENOENT: llvm-[[TOOL]]{{(\.exe)?}}: error: '{{.*}}.blah': {{[Nn]}}o such file or directory
 
 # Test case with no input file.
 RUN: not llvm-readobj 2>&1 | FileCheck %s --check-prefix=NO-FILE
diff --git a/llvm/test/tools/llvm-readobj/thin-archive.test b/llvm/test/tools/llvm-readobj/thin-archive.test
index 17782293688e..d13ae23e1b89 100644
--- a/llvm/test/tools/llvm-readobj/thin-archive.test
+++ b/llvm/test/tools/llvm-readobj/thin-archive.test
@@ -60,7 +60,7 @@
 # MISSING:     File: {{.*}}1.o
 # MISSING:     Format: elf64-x86-64
 # MISSING-NOT: File: {{.*}}3.o
-# ERR: error: '{{.*}}.a': '{{.*}}2.o': {{.*}}{{[Nn]}}o such file or directory
+# ERR: error: '{{.*}}.a': '{{.*}}2.o': {{[Nn]}}o such file or directory
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-size/no-input.test b/llvm/test/tools/llvm-size/no-input.test
index 6be4a24171e8..37e5745bf177 100644
--- a/llvm/test/tools/llvm-size/no-input.test
+++ b/llvm/test/tools/llvm-size/no-input.test
@@ -1,7 +1,7 @@
 ## Show that llvm-size emits an error if passed in a non-existent file.
 
 # RUN: not llvm-size %t.blah 2>&1 | FileCheck %s -DFILE=%t.blah --check-prefix=ENOENT
-# ENOENT: {{.*}}llvm-size{{.*}}: error: '[[FILE]]': {{.*}}{{[Nn]}}o such file or directory
+# ENOENT: {{.*}}llvm-size{{.*}}: error: '[[FILE]]': {{[Nn]}}o such file or directory
 
 ## Show that llvm-size reads a.out if not passed any file.
 
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/missing_pdb.test b/llvm/test/tools/llvm-symbolizer/pdb/missing_pdb.test
index 8948abe17734..b83b46a75213 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/missing_pdb.test
+++ b/llvm/test/tools/llvm-symbolizer/pdb/missing_pdb.test
@@ -4,7 +4,7 @@ RUN: FileCheck --check-prefix=ERROR %s < %t.err
 
 llvm-symbolizer should print one error and two unknown line info records.
 
-ERROR: LLVMSymbolizer: error reading file: {{.*}}: {{.*}}{{N|n}}o such file or directory
+ERROR: LLVMSymbolizer: error reading file: {{.*}}: {{N|n}}o such file or directory
 ERROR-NOT: error reading file
 
 CHECK: ??
diff --git a/llvm/test/tools/llvm-xray/X86/no-such-file.txt b/llvm/test/tools/llvm-xray/X86/no-such-file.txt
index d647b528ccad..69fc43f9625b 100644
--- a/llvm/test/tools/llvm-xray/X86/no-such-file.txt
+++ b/llvm/test/tools/llvm-xray/X86/no-such-file.txt
@@ -1,4 +1,4 @@
 ; RUN: not llvm-xray extract no-such-file 2>&1 | FileCheck %s
 
 ; CHECK: llvm-xray: Cannot extract instrumentation map from 'no-such-file'.
-; CHECK-NEXT: {{.*}}{{[Nn]}}o such file or directory
+; CHECK-NEXT: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/obj2yaml/ELF/offset.yaml b/llvm/test/tools/obj2yaml/ELF/offset.yaml
index 1eb464c9839b..47970a07535c 100644
--- a/llvm/test/tools/obj2yaml/ELF/offset.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/offset.yaml
@@ -36,17 +36,17 @@
 #   BASIC-NEXT:     Type:         SHT_PROGBITS
 #   BASIC-NEXT:     AddressAlign: 0x100000000
 #   BASIC-NEXT:     Offset:       0x210
-# HEADERS-NEXT: SectionHeaderTable:
-# HEADERS-NEXT:   Sections:
-# HEADERS-NEXT:     - Name:            .bar4
-# HEADERS-NEXT:     - Name:            .bar3
-# HEADERS-NEXT:     - Name:            .bar2
-# HEADERS-NEXT:     - Name:            .bar1
-# HEADERS-NEXT:     - Name:            .foo3
-# HEADERS-NEXT:     - Name:            .foo2
-# HEADERS-NEXT:     - Name:            .foo1
-# HEADERS-NEXT:     - Name:            .strtab
-# HEADERS-NEXT:     - Name:            .shstrtab
+# HEADERS-NEXT:   - Type: SectionHeaderTable
+# HEADERS-NEXT:     Sections:
+# HEADERS-NEXT:       - Name: .bar4
+# HEADERS-NEXT:       - Name: .bar3
+# HEADERS-NEXT:       - Name: .bar2
+# HEADERS-NEXT:       - Name: .bar1
+# HEADERS-NEXT:       - Name: .foo3
+# HEADERS-NEXT:       - Name: .foo2
+# HEADERS-NEXT:       - Name: .foo1
+# HEADERS-NEXT:       - Name: .strtab
+# HEADERS-NEXT:       - Name: .shstrtab
 # BASIC-NEXT: ...
 
 --- !ELF
@@ -103,18 +103,18 @@ Sections:
     Type:         SHT_PROGBITS
     AddressAlign: 0x100000000
     Offset:       0x210
-SectionHeaderTable:
-  Sections:
+  - Type: SectionHeaderTable
+    Sections:
 ## By default we have the same order of sections as defined by the "Sections" key.
-    - Name: [[SEC1=.foo1]]
-    - Name: [[SEC2=.foo2]]
-    - Name: [[SEC3=.foo3]]
-    - Name: [[SEC4=.bar1]]
-    - Name: [[SEC5=.bar2]]
-    - Name: [[SEC6=.bar3]]
-    - Name: [[SEC7=.bar4]]
-    - Name: .strtab
-    - Name: .shstrtab
+      - Name: [[SEC1=.foo1]]
+      - Name: [[SEC2=.foo2]]
+      - Name: [[SEC3=.foo3]]
+      - Name: [[SEC4=.bar1]]
+      - Name: [[SEC5=.bar2]]
+      - Name: [[SEC6=.bar3]]
+      - Name: [[SEC7=.bar4]]
+      - Name: .strtab
+      - Name: .shstrtab
 
 ## In this case we change the order of sections in the section header table.
 ## Check that we still dump offsets correctly.
diff --git a/llvm/test/tools/obj2yaml/invalid_input_file.test b/llvm/test/tools/obj2yaml/invalid_input_file.test
index 583e4677b2a8..0f1bac1be01a 100644
--- a/llvm/test/tools/obj2yaml/invalid_input_file.test
+++ b/llvm/test/tools/obj2yaml/invalid_input_file.test
@@ -1,3 +1,3 @@
 # RUN: not obj2yaml  %p/path/does/not/exist 2>&1 | FileCheck %s
 
-# CHECK: Error reading file: {{.*}}/path/does/not/exist: {{.*}}{{[Nn]}}o such file or directory
+# CHECK: Error reading file: {{.*}}/path/does/not/exist: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/yaml2obj/ELF/section-headers-exclude.yaml b/llvm/test/tools/yaml2obj/ELF/section-headers-exclude.yaml
index 45b339853cf1..ce19696a35a9 100644
--- a/llvm/test/tools/yaml2obj/ELF/section-headers-exclude.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/section-headers-exclude.yaml
@@ -32,13 +32,13 @@ Sections:
     Type: SHT_PROGBITS
   - Name: .bar
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: [[INCLUDED]]
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: [[EXCLUDED]]
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: [[INCLUDED]]
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: [[EXCLUDED]]
 
 ## Check we report an error when a section is in both the "Sections" and "Excluded" lists at the same time.
 ## Also check that we report an error if a section is missing from the lists.
@@ -66,13 +66,14 @@ FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_REL
-SectionHeaderTable:
-  Sections:
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .strtab
-    - Name: .strtab
+Sections:
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .strtab
+      - Name: .strtab
 
 ## Check that we are able to exclude all sections, except the implicit
 ## null section, with the use of the "Excluded" key.
@@ -91,11 +92,12 @@ FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_REL
-SectionHeaderTable:
-  Sections: []
-  Excluded:
-    - Name: .strtab
-    - Name: .shstrtab
+Sections:
+  - Type: SectionHeaderTable
+    Sections: []
+    Excluded:
+      - Name: .strtab
+      - Name: .shstrtab
 
 ## Case B: the "Sections" key is not present.
 # RUN: yaml2obj %s --docnum=4 -o %t4
@@ -106,10 +108,11 @@ FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_REL
-SectionHeaderTable:
-  Excluded:
-    - Name: .strtab
-    - Name: .shstrtab
+Sections:
+  - Type: SectionHeaderTable
+    Excluded:
+      - Name: .strtab
+      - Name: .shstrtab
 
 ## Check how we handle cases when a section is excluded, but its section index is needed.
 ## The general rule is: when a section is explicitly linked with another section, which is
@@ -134,13 +137,13 @@ Sections:
     Link: .bar
   - Name: .bar
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: .foo
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .bar
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .foo
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .bar
 
 ## Case B.1: check we report an error when a symbol table section has a Link field which
 ##           points to an excluded section.
@@ -160,13 +163,13 @@ Sections:
     Link:  .foo
   - Name:  .foo
     Type:  SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: [[NAME]]
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .foo
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: [[NAME]]
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .foo
 
 ## Case B.2: check we do not link .dynsym with .dynstr implicitly when the latter is excluded.
 # RUN: yaml2obj %s --docnum=7 -o %t5
@@ -185,13 +188,13 @@ Sections:
     Type:  SHT_DYNSYM
   - Name:  .dynstr
     Type:  SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: .dynsym
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .dynstr
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .dynsym
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .dynstr
 
 ## Case B.3: check we do not link .symtab with .strtab implicitly when the latter is excluded.
 # RUN: yaml2obj %s --docnum=8 -o %t6
@@ -210,12 +213,12 @@ Sections:
     Type:  SHT_SYMTAB
   - Name:  .strtab
     Type:  SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: .symtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .strtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .symtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .strtab
 
 ## Case C: check we report an error when a debug section has a Link field which
 ##         points to an excluded section.
@@ -231,12 +234,12 @@ Sections:
   - Name:  .debug_unknown
     Type:  SHT_PROGBITS
     Link:  .strtab
-SectionHeaderTable:
-  Sections:
-    - Name: .debug_unknown
-    - Name: .shstrtab
-  Excluded:
-    - Name: .strtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .debug_unknown
+      - Name: .shstrtab
+    Excluded:
+      - Name: .strtab
 
 ## Case D.1: check we report an error when a relocatable section has an Info field which
 ##           points to an excluded section.
@@ -253,12 +256,12 @@ Sections:
     Type: SHT_RELA
     Info: .strtab
     Relocations: []
-SectionHeaderTable:
-  Sections:
-    - Name: .rela
-    - Name: .shstrtab
-  Excluded:
-    - Name: .strtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .rela
+      - Name: .shstrtab
+    Excluded:
+      - Name: .strtab
 
 ## Case D.2: check we report an error when the SHT_REL[A] section is linked
 ##           with an excluded section explicitly.
@@ -277,13 +280,13 @@ Sections:
     Relocations: []
   - Name: .symtab
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: .rela
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .symtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .rela
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .symtab
 
 ## Case E: check we report an error when a symbol references an excluded section.
 # RUN: not yaml2obj %s --docnum=12 -o /dev/null 2>&1 | \
@@ -299,17 +302,17 @@ FileHeader:
 Sections:
   - Name: .foo
     Type: SHT_PROGBITS
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .symtab
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .foo
 Symbols:
   - Name:    foo
     Type:    STT_OBJECT
     Section: .foo
-SectionHeaderTable:
-  Sections:
-    - Name: .symtab
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .foo
 
 ## Case F.1: check we report an error when a group section
 ##           contains an excluded section member.
@@ -326,12 +329,12 @@ Sections:
     Type: SHT_GROUP
     Members:
       - SectionOrType: .strtab
-SectionHeaderTable:
-  Sections:
-    - Name: .group
-    - Name: .shstrtab
-  Excluded:
-    - Name: .strtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .group
+      - Name: .shstrtab
+    Excluded:
+      - Name: .strtab
 
 ## Case F.2: check we report an error when the group section is linked
 ##           to an excluded section explicitly.
@@ -350,13 +353,13 @@ Sections:
     Members: []
   - Name: .symtab
     Type: SHT_SYMTAB
-SectionHeaderTable:
-  Sections:
-    - Name: .group
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .symtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .group
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .symtab
 
 ## Case G: check we do not link SHT_LLVM_CALL_GRAPH_PROFILE/SHT_LLVM_ADDRSIG/SHT_GROUP/SHT_REL[A] sections
 ##         with .symtab implicitly when the latter is excluded.
@@ -389,16 +392,16 @@ Sections:
     Relocations: []
   - Name: .symtab
     Type: SHT_SYMTAB
-SectionHeaderTable:
-  Sections:
-    - Name: .cgp
-    - Name: .llvm_addrsig
-    - Name: .group
-    - Name: .rela
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .symtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .cgp
+      - Name: .llvm_addrsig
+      - Name: .group
+      - Name: .rela
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .symtab
 
 ## Case H: check we do not link SHT_HASH/SHT_GNU_HASH sections with .dynsym
 ##         implicitly when the latter is excluded.
@@ -423,14 +426,14 @@ Sections:
     Content: ""
   - Name: .dynsym
     Type: SHT_DYNSYM
-SectionHeaderTable:
-  Sections:
-    - Name: .hash
-    - Name: .gnu_hash
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .dynsym
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .hash
+      - Name: .gnu_hash
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .dynsym
 
 ## Case I: document the case when an excluded section is explicitly linked to another excluded section.
 ##         We report an error in this case, because:
@@ -454,13 +457,13 @@ Sections:
   - Name: .bar
     Type: SHT_PROGBITS
     Link: .foo
-SectionHeaderTable:
-  Sections:
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .foo
-    - Name: .bar
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .foo
+      - Name: .bar
 
 ## Check we set e_shstrndx field to 0 when the section header string table is excluded.
 ## Check that the e_shnum field is adjusted properly when a section is removed.
@@ -475,22 +478,24 @@ FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_REL
-SectionHeaderTable:
-  Sections:
-    - Name: .strtab
-  Excluded:
-    - Name: .shstrtab
+Sections:
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .strtab
+    Excluded:
+      - Name: .shstrtab
 
 ## Check we do not allow using "Excluded" together with "NoHeaders".
 # RUN: not yaml2obj %s --docnum=19 -DNOHEADERS=true -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOHEADERS
 # RUN: not yaml2obj %s --docnum=19 -DNOHEADERS=false -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOHEADERS
-# NOHEADERS: NoHeaders can't be used together with Sections/Excluded
+# NOHEADERS: NoHeaders can't be used together with Offset/Sections/Excluded
 
 --- !ELF
 FileHeader:
   Class: ELFCLASS64
   Data:  ELFDATA2LSB
   Type:  ET_REL
-SectionHeaderTable:
-  NoHeaders: [[NOHEADERS]]
-  Excluded:  []
+Sections:
+  - Type: SectionHeaderTable
+    NoHeaders: [[NOHEADERS]]
+    Excluded:  []
diff --git a/llvm/test/tools/yaml2obj/ELF/section-headers.yaml b/llvm/test/tools/yaml2obj/ELF/section-headers.yaml
index 7eadb4a881b3..5fe029f42e75 100644
--- a/llvm/test/tools/yaml2obj/ELF/section-headers.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/section-headers.yaml
@@ -34,13 +34,13 @@ Sections:
   - Name: .section.foo
     Type: SHT_PROGBITS
     Size: 0x40
-SectionHeaderTable:
-  Sections:
-    - Name: [[SEC1]]
-    - Name: [[SEC2]]
-    - Name: [[SEC3]]
-    - Name: .strtab
-    - Name: .shstrtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: [[SEC1]]
+      - Name: [[SEC2]]
+      - Name: [[SEC3]]
+      - Name: .strtab
+      - Name: .shstrtab
 
 ## Show we are able to reorder sections.
 # RUN: yaml2obj %s -o %t2 -DSEC3=".section (1)" -DSEC2=".section (2)" -DSEC1=".section.foo"
@@ -93,8 +93,8 @@ FileHeader:
 Sections:
   - Name: .foo
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  Sections: []
+  - Type: SectionHeaderTable
+    Sections: []
 
 ## Test that we are able to use "NoHeaders" property to produce an empty section header table.
 # RUN: yaml2obj %s --docnum=3 -DNOHEADERS=true -o %t3.1
@@ -113,8 +113,8 @@ FileHeader:
 Sections:
   - Name: .foo
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  NoHeaders: [[NOHEADERS]]
+  - Type: SectionHeaderTable
+    NoHeaders: [[NOHEADERS]]
 
 ## Test that we are able to set NoHeaders to false. In this case the tool produces an output
 ## as if there were no `SectionHeaderTable` key at all.
@@ -137,7 +137,7 @@ SectionHeaderTable:
 # RUN: not yaml2obj %s --docnum=4 -DNOHEADERS=true -o /dev/null 2>&1 | FileCheck %s --check-prefix=SECTIONS-NO-HEADERS
 # RUN: not yaml2obj %s --docnum=4 -DNOHEADERS=false -o /dev/null 2>&1 | FileCheck %s --check-prefix=SECTIONS-NO-HEADERS
 
-# SECTIONS-NO-HEADERS: error: NoHeaders can't be used together with Sections/Excluded
+# SECTIONS-NO-HEADERS: error: NoHeaders can't be used together with Offset/Sections/Excluded
 
 --- !ELF
 FileHeader:
@@ -147,12 +147,12 @@ FileHeader:
 Sections:
   - Name: .foo
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  Sections:  []
-  NoHeaders: [[NOHEADERS]]
+  - Type: SectionHeaderTable
+    Sections:  []
+    NoHeaders: [[NOHEADERS]]
 
 ## Check that we do not allow an empty SectionHeaderTable tag and suggest to use an explicit syntax instead.
-# RUN: not yaml2obj %s --docnum=5 -DVAL="" -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO-VALUE
+# RUN: not yaml2obj %s --docnum=5 -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO-VALUE
 
 # NO-VALUE: SectionHeaderTable can't be empty. Use 'NoHeaders' key to drop the section header table
 
@@ -164,12 +164,7 @@ FileHeader:
 Sections:
   - Name: .foo
     Type: SHT_PROGBITS
-SectionHeaderTable: [[VAL]]
-
-## An empty mapping is also not allowed for the SectionHeaderTable tag.
-# RUN: not yaml2obj %s --docnum=5 -DVAL="[]" -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOT-A-MAPPING
-
-# NOT-A-MAPPING: error: not a mapping
+  - Type: SectionHeaderTable
 
 ## Test that we are still able to override e_shoff, e_shnum and e_shstrndx
 ## fields even when we do not produce section headers.
@@ -188,8 +183,9 @@ FileHeader:
   EShOff:    0x2
   EShNum:    0x3
   EShStrNdx: 0x4
-SectionHeaderTable:
-  NoHeaders: true
+Sections:
+  - Type:      SectionHeaderTable
+    NoHeaders: true
 
 ## Check that section indices are updated properly in other places when we
 ## reorder sections in the section header table.
@@ -233,15 +229,15 @@ Sections:
   - Name: .another.2
     Link: .bar
     Type: SHT_PROGBITS
-SectionHeaderTable:
-  Sections:
-    - Name: [[SEC1]]
-    - Name: [[SEC2]]
-    - Name: .another.1
-    - Name: .another.2
-    - Name: .symtab
-    - Name: .strtab
-    - Name: .shstrtab
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: [[SEC1]]
+      - Name: [[SEC2]]
+      - Name: .another.1
+      - Name: .another.2
+      - Name: .symtab
+      - Name: .strtab
+      - Name: .shstrtab
 Symbols:
   - Name:    foo
     Section: .foo
@@ -275,5 +271,142 @@ Sections:
 ## for writing the section header table.
     Size:   0xFF
     Offset: 0x100
-SectionHeaderTable:
-  NoHeaders: true
+  - Type:      SectionHeaderTable
+    NoHeaders: true
+
+## Check we do not allow using "Offset" together with "NoHeaders".
+# RUN: not yaml2obj %s --docnum=9 -DNOHEADERS=true -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO-HEADERS-OFFSET
+# RUN: not yaml2obj %s --docnum=9 -DNOHEADERS=false -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO-HEADERS-OFFSET
+
+# NO-HEADERS-OFFSET: error: NoHeaders can't be used together with Offset/Sections/Excluded
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_REL
+Sections:
+  - Name: .foo
+    Type: SHT_PROGBITS
+  - Type:      SectionHeaderTable
+    Offset:    0x1000
+    NoHeaders: [[NOHEADERS]]
+
+## Check we can place the section header table before other sections.
+
+# RUN: yaml2obj %s --docnum=10 -o %t10
+# RUN: llvm-readelf --file-headers --sections %t10 | FileCheck %s --check-prefix=PLACE-BEFORE
+
+# PLACE-BEFORE: Start of section headers:  64 (bytes into file)
+# PLACE-BEFORE: Size of section headers:   64 (bytes)
+# PLACE-BEFORE: Number of section headers: 4
+
+# PLACE-BEFORE: Section Headers:
+# PLACE-BEFORE:   [Nr] Name Type     Address          Off    Size
+## 0x140 == Start of section headers (64) + Size of section headers (64) * Number of section headers (4);
+# PLACE-BEFORE:   [ 1] .foo PROGBITS 0000000000000000 000140 000000
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_REL
+Sections:
+  - Type:   SectionHeaderTable
+    Offset: [[OFFSET=<none>]]
+    Sections:
+      - Name: .foo
+      - Name: .strtab
+      - Name: .shstrtab
+  - Name: .foo
+    Type: SHT_PROGBITS
+
+## Check we can use the Offset key to place the section header table at a different location.
+
+# RUN: yaml2obj %s --docnum=10 -DOFFSET=0x1000 -o %t10.offset
+# RUN: llvm-readelf --file-headers --sections %t10.offset | \
+# RUN:   FileCheck %s --check-prefix=PLACE-BEFORE-OFFSET
+
+# PLACE-BEFORE-OFFSET: Start of section headers:          4096 (bytes into file)
+# PLACE-BEFORE-OFFSET: Size of section headers:           64 (bytes)
+# PLACE-BEFORE-OFFSET: Number of section headers:         4
+
+# PLACE-BEFORE-OFFSET: Section Headers:
+# PLACE-BEFORE-OFFSET:   [Nr] Name Type     Address          Off    Size
+## 0x1100 == Start of section headers (0x1000) + Size of section headers (64) * Number of section headers (4);
+# PLACE-BEFORE-OFFSET:   [ 1] .foo PROGBITS 0000000000000000 001100 000000
+
+## Check we can place the section header table somewhere in the middle of the sections list.
+
+# RUN: yaml2obj %s --docnum=11 -o %t11
+# RUN: llvm-readelf --sections %t11 | FileCheck %s --check-prefix=PLACE-AT-MIDDLE
+
+# PLACE-AT-MIDDLE: There are 5 section headers, starting at offset 0x140:
+# PLACE-AT-MIDDLE:      [Nr] Name      Type     Address          Off    Size
+# PLACE-AT-MIDDLE:      [ 1] .foo      PROGBITS 0000000000000000 000040 000100
+## The section header table is placed here, at 0x40 + 0x100 == 0x140.
+## The size of the table is 5 * 64 == 320 == 0x140.
+## 0x140 + 0x140 == offset of the .bar section.
+# PLACE-AT-MIDDLE-NEXT: [ 2] .bar      PROGBITS 0000000000000000 000280 000200
+# PLACE-AT-MIDDLE-NEXT: [ 3] .strtab   STRTAB   0000000000000000 000480 000001
+# PLACE-AT-MIDDLE-NEXT: [ 4] .shstrtab STRTAB   0000000000000000 000481 00001d
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_REL
+Sections:
+  - Name: .foo
+    Type: SHT_PROGBITS
+    Size: 0x100
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .foo
+      - Name: .bar
+      - Name: .strtab
+      - Name: .shstrtab
+  - Name: .bar
+    Type: SHT_PROGBITS
+    Size: 0x200
+
+## Check we don`t allow having multiple SectionHeaderTable chunks.
+
+# RUN: not yaml2obj %s --docnum=12 -o /dev/null 2>&1 | FileCheck %s --check-prefix=MULTIPLE
+
+# MULTIPLE: error: multiple section header tables are not allowed
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_REL
+Sections:
+  - Type:     SectionHeaderTable
+    Sections: []
+  - Type:     SectionHeaderTable
+    Sections: []
+
+## Check that we place implicit sections before the section header table
+## when it is placed at the end explicitly.
+
+# RUN: yaml2obj %s --docnum=13 -o %t13
+# RUN: llvm-readelf --sections %t13 | FileCheck %s --check-prefix=IMPLICIT
+
+# IMPLICIT:      There are 3 section headers, starting at offset 0x58:
+# IMPLICIT:      [Nr] Name      Type   Address          Off    Size
+# IMPLICIT:      [ 1] .strtab   STRTAB 0000000000000000 000040 000001
+# IMPLICIT-NEXT: [ 2] .shstrtab STRTAB 0000000000000000 000041 000013
+## The section header table is placed here, at 0x58.
+## alignTo(8, 0x41 + 0x13) == 0x58.
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_REL
+Sections:
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .strtab
+      - Name: .shstrtab
diff --git a/llvm/test/tools/yaml2obj/ELF/verdef-section.yaml b/llvm/test/tools/yaml2obj/ELF/verdef-section.yaml
index 65490da101c6..fe311c302ae2 100644
--- a/llvm/test/tools/yaml2obj/ELF/verdef-section.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/verdef-section.yaml
@@ -241,10 +241,10 @@ Sections:
     Type: SHT_GNU_verdef
   - Name: .dynstr
     Type: SHT_STRTAB
-SectionHeaderTable:
-  Sections:
-    - Name: .gnu.version_d
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .dynstr
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .gnu.version_d
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .dynstr
diff --git a/llvm/test/tools/yaml2obj/ELF/verneed-section.yaml b/llvm/test/tools/yaml2obj/ELF/verneed-section.yaml
index 2fbf83a69b4e..7f378476e06f 100644
--- a/llvm/test/tools/yaml2obj/ELF/verneed-section.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/verneed-section.yaml
@@ -182,10 +182,10 @@ Sections:
     Type: SHT_GNU_verneed
   - Name: .dynstr
     Type: SHT_STRTAB
-SectionHeaderTable:
-  Sections:
-    - Name: .gnu.version_r
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .dynstr
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .gnu.version_r
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .dynstr
diff --git a/llvm/test/tools/yaml2obj/ELF/versym-section.yaml b/llvm/test/tools/yaml2obj/ELF/versym-section.yaml
index 24a0916a0ef8..4be29c20f172 100644
--- a/llvm/test/tools/yaml2obj/ELF/versym-section.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/versym-section.yaml
@@ -194,10 +194,10 @@ Sections:
     Type: SHT_GNU_versym
   - Name: .dynsym
     Type: SHT_DYNSYM
-SectionHeaderTable:
-  Sections:
-    - Name: .gnu.version
-    - Name: .strtab
-    - Name: .shstrtab
-  Excluded:
-    - Name: .dynsym
+  - Type: SectionHeaderTable
+    Sections:
+      - Name: .gnu.version
+      - Name: .strtab
+      - Name: .shstrtab
+    Excluded:
+      - Name: .dynsym
diff --git a/llvm/test/tools/yaml2obj/output-file.yaml b/llvm/test/tools/yaml2obj/output-file.yaml
index e3df91a280f6..e900291fc4c0 100644
--- a/llvm/test/tools/yaml2obj/output-file.yaml
+++ b/llvm/test/tools/yaml2obj/output-file.yaml
@@ -9,7 +9,7 @@
 
 # RUN: not yaml2obj -o %p/path/does/not/exist 2>&1 | FileCheck %s
 
-# CHECK: yaml2obj: error: failed to open '{{.*}}/path/does/not/exist': {{.*}}{{[Nn]}}o such file or directory
+# CHECK: yaml2obj: error: failed to open '{{.*}}/path/does/not/exist': {{[Nn]}}o such file or directory
 
 !ELF
 FileHeader:
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 7bd7575dab41..9aed3526b0aa 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -696,6 +696,14 @@ static int error(const Twine &Error, const Twine &Context) {
   return 1;
 }
 
+static Expected<Triple> readTargetTriple(StringRef FileName) {
+  auto ErrOrObj = object::ObjectFile::createObjectFile(FileName);
+  if (!ErrOrObj)
+    return ErrOrObj.takeError();
+
+  return ErrOrObj->getBinary()->makeTriple();
+}
+
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
 
@@ -706,17 +714,36 @@ int main(int argc, char **argv) {
   llvm::InitializeAllTargets();
   llvm::InitializeAllAsmPrinters();
 
+  std::vector<std::string> DWOFilenames = InputFiles;
+  for (const auto &ExecFilename : ExecFilenames) {
+    auto DWOs = getDWOFilenames(ExecFilename);
+    if (!DWOs) {
+      logAllUnhandledErrors(DWOs.takeError(), WithColor::error());
+      return 1;
+    }
+    DWOFilenames.insert(DWOFilenames.end(),
+                        std::make_move_iterator(DWOs->begin()),
+                        std::make_move_iterator(DWOs->end()));
+  }
+
+  if (DWOFilenames.empty())
+    return 0;
+
   std::string ErrorStr;
   StringRef Context = "dwarf streamer init";
 
-  Triple TheTriple("x86_64-linux-gnu");
+  auto ErrOrTriple = readTargetTriple(DWOFilenames.front());
+  if (!ErrOrTriple) {
+    logAllUnhandledErrors(ErrOrTriple.takeError(), WithColor::error());
+    return 1;
+  }
 
   // Get the target.
   const Target *TheTarget =
-      TargetRegistry::lookupTarget("", TheTriple, ErrorStr);
+      TargetRegistry::lookupTarget("", *ErrOrTriple, ErrorStr);
   if (!TheTarget)
     return error(ErrorStr, Context);
-  std::string TripleName = TheTriple.getTriple();
+  std::string TripleName = ErrOrTriple->getTriple();
 
   // Create all the MC Objects.
   std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TripleName));
@@ -731,7 +758,7 @@ int main(int argc, char **argv) {
 
   MCObjectFileInfo MOFI;
   MCContext MC(MAI.get(), MRI.get(), &MOFI);
-  MOFI.InitMCObjectFileInfo(TheTriple, /*PIC*/ false, MC);
+  MOFI.InitMCObjectFileInfo(*ErrOrTriple, /*PIC*/ false, MC);
 
   std::unique_ptr<MCSubtargetInfo> MSTI(
       TheTarget->createMCSubtargetInfo(TripleName, "", ""));
@@ -766,25 +793,13 @@ int main(int argc, char **argv) {
   }
 
   std::unique_ptr<MCStreamer> MS(TheTarget->createMCObjectStreamer(
-      TheTriple, MC, std::unique_ptr<MCAsmBackend>(MAB),
+      *ErrOrTriple, MC, std::unique_ptr<MCAsmBackend>(MAB),
       MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(MCE), *MSTI,
       MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
       /*DWARFMustBeAtTheEnd*/ false));
   if (!MS)
     return error("no object streamer for target " + TripleName, Context);
 
-  std::vector<std::string> DWOFilenames = InputFiles;
-  for (const auto &ExecFilename : ExecFilenames) {
-    auto DWOs = getDWOFilenames(ExecFilename);
-    if (!DWOs) {
-      logAllUnhandledErrors(DWOs.takeError(), WithColor::error());
-      return 1;
-    }
-    DWOFilenames.insert(DWOFilenames.end(),
-                        std::make_move_iterator(DWOs->begin()),
-                        std::make_move_iterator(DWOs->end()));
-  }
-
   if (auto Err = write(*MS, DWOFilenames)) {
     logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp
index d92e2254d9b3..c85e2653b655 100644
--- a/llvm/tools/obj2yaml/elf2yaml.cpp
+++ b/llvm/tools/obj2yaml/elf2yaml.cpp
@@ -382,13 +382,18 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
                cast<ELFYAML::Section>(B.get())->OriginalSecNdx;
       });
   if (!SectionsSorted) {
-    Y->SectionHeaders.emplace();
-    Y->SectionHeaders->Sections.emplace();
+    std::unique_ptr<ELFYAML::SectionHeaderTable> SHT =
+        std::make_unique<ELFYAML::SectionHeaderTable>(/*IsImplicit=*/false);
+    SHT->Sections.emplace();
     for (ELFYAML::Section *S : OriginalOrder)
-      Y->SectionHeaders->Sections->push_back({S->Name});
+      SHT->Sections->push_back({S->Name});
+    Chunks.push_back(std::move(SHT));
   }
 
   llvm::erase_if(Chunks, [this, &Y](const std::unique_ptr<ELFYAML::Chunk> &C) {
+    if (isa<ELFYAML::SectionHeaderTable>(*C.get()))
+      return false;
+
     const ELFYAML::Section &S = cast<ELFYAML::Section>(*C.get());
     return !shouldPrintSection(S, Sections[S.OriginalSecNdx], Y->DWARF);
   });
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index 8026a3a102be..cd97733ce984 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -933,7 +933,8 @@ class RuleMatcher : public Matcher {
                                 StringRef ParentSymbolicName) {
     std::string ParentName(ParentSymbolicName);
     if (ComplexSubOperands.count(SymbolicName)) {
-      auto RecordedParentName = ComplexSubOperandsParentName[SymbolicName];
+      const std::string &RecordedParentName =
+          ComplexSubOperandsParentName[SymbolicName];
       if (RecordedParentName != ParentName)
         return failedImport("Error: Complex suboperand " + SymbolicName +
                             " referenced by different operands: " +
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index ae9f81d043f5..9ea1bc5a3587 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -475,6 +475,38 @@ def Linalg_SliceOp : Linalg_Op<"slice", [
   let hasFolder = 1;
 }
 
+def Linalg_SimplePadOp : Linalg_Op<"simple_pad", [NoSideEffect]> {
+  let summary = "TODO: replace with pad_tensors when ready.";
+
+  let description = [{
+    `linalg.simple_pad` is a tmp placeholder for padding and packing on tensors.
+    Its semantics are to pad a partially dynamic tensor to a fully static tensor
+    where the static sizes are assumed to be greater than the dynamic sizes. The
+    op perforrms "high" padding (i.e. it adds trailing padding values until the 
+    desired size is met).
+  }];
+
+  let arguments = (ins AnyRankedTensor:$tensor, AnyType:$padding);
+  let results = (outs AnyRankedTensor:$result);
+
+  // TODO: verify all static result, some dynamic input, static shapes match,
+  // element types match, ranks match etc. Use pad_tensors when ready but for
+  // now just let it ne fully specified by traits.
+  let verifier = ?;
+
+  let extraClassDeclaration = [{
+    RankedTensorType getSourceType() {
+      return tensor().getType().cast<RankedTensorType>(); }
+    RankedTensorType getResultType() {
+      return getResult().getType().cast<RankedTensorType>(); }
+   }];
+
+  let assemblyFormat = [{
+    $tensor `pad` $padding attr-dict `:`
+      type($tensor) `to` type($result) `pad` type($padding)
+  }];
+}
+
 def Linalg_YieldOp : Linalg_Op<"yield", [NoSideEffect, ReturnLike, Terminator]>,
     Arguments<(ins Variadic<AnyType>:$values)> {
   let summary = "Linalg yield operation";
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Hoisting.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Hoisting.h
index ed585d1f5cf5..4d44b3717991 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Hoisting.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Hoisting.h
@@ -11,8 +11,10 @@
 
 namespace mlir {
 class FuncOp;
+struct LogicalResult;
 
 namespace linalg {
+class SimplePadOp;
 
 /// Hoist alloc/dealloc pairs and alloca op out of immediately enclosing
 /// scf::ForOp if both conditions are true:
@@ -40,6 +42,44 @@ void hoistRedundantVectorTransfers(FuncOp func);
 /// instead of buffers.
 void hoistRedundantVectorTransfersOnTensor(FuncOp func);
 
+/// Mechanically hoist padding operations on tensors by `nLoops` into a new,
+/// generally larger tensor. This achieves packing of multiple padding ops into
+/// a larger tensor. On success, `simplePadOp` is replaced by the cloned version
+/// in the packing loop so the caller can continue reasoning about the padding
+/// operation.
+///
+/// Example in pseudo-mlir:
+/// =======================
+///
+/// If hoistPaddingOnTensors is called with `nLoops` = 2 on the following IR.
+/// ```
+///    scf.for (%i, %j, %k)
+///      %st0 = subtensor f(%i, %k) : ... to tensor<?x?xf32>
+///      %0 = linalg.simple_pad %st0 pad %pad :
+///             tensor<?x?xf32> to tensor<4x8xf32>
+///      compute(%0)
+/// ```
+///
+/// IR resembling the following is produced:
+///
+/// ```
+///    scf.for (%i) {
+///      %packed_init = linalg.init_tensor range(%j) : tensor<?x4x8xf32>
+///      %packed = scf.for (%k) iter_args(%p : %packed_init)
+///        %st0 = subtensor f(%i, %k) : ... to tensor<?x?xf32>
+///        %0 = linalg.simple_pad %st0 pad %pad :
+///               tensor<?x?xf32> to tensor<4x8xf32>
+///        scf.yield %1: tensor<?x4x8xf32>
+///      } -> tensor<?x4x8xf32>
+///      scf.for (%j, %k) {
+///        %st0 = subtensor %packed [%k, 0, 0][1, 4, 8][1, 1, 1] :
+///                 tensor<?x4x8xf32> to tensor<4x8xf32>
+///        compute(%st0)
+///      }
+///    }
+/// ```
+LogicalResult hoistPaddingOnTensors(SimplePadOp &simplePadOp, unsigned nLoops);
+
 } // namespace linalg
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 611ab6867372..f359992e5ff1 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -345,6 +345,9 @@ enum class LinalgTilingLoopType {
 using TileSizeComputationFunction =
     std::function<SmallVector<Value, 4>(OpBuilder &, Operation *)>;
 
+using PaddingValueComputationFunction =
+    std::function<Value(OpBuilder &, Operation *)>;
+
 struct LinalgTilingOptions {
   /// Computation function that returns the tile sizes for each operation.
   /// Delayed construction of constant tile sizes should occur to interoperate
@@ -393,6 +396,18 @@ struct LinalgTilingOptions {
     distribution = std::move(distributionOptions);
     return *this;
   }
+
+  /// Computation function that returns a padding value to use when padding to
+  /// force static sizes. When `paddingValueComputationFunction` is set, padding
+  /// operations are introduced, that guarantee the underlying op is statically
+  /// shaped and can thus be vectorized.
+  PaddingValueComputationFunction paddingValueComputationFunction = nullptr;
+
+  LinalgTilingOptions &
+  setPaddingValueComputationFunction(PaddingValueComputationFunction fun) {
+    paddingValueComputationFunction = std::move(fun);
+    return *this;
+  }
 };
 
 /// Canonicalization patterns relevant to apply after tiling patterns. These are
@@ -403,6 +418,11 @@ getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx);
 void populateLinalgTilingCanonicalizationPatterns(
     OwningRewritePatternList &patterns, MLIRContext *ctx);
 
+/// Base pattern that applied the tiling transformation specified by `options`.
+/// Abort and return failure in 2 cases:
+///   1. if the tiling specification is invalid and tiling fails to occur.
+///   2. if tiling occurs but `options.paddingValueComputationFunction` is set
+///      and some operand shape cannot be bounded statically.
 struct LinalgBaseTilingPattern : public RewritePattern {
   // Entry point to match any LinalgOp OpInterface.
   LinalgBaseTilingPattern(LinalgTilingOptions options,
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index ce1907cb6435..08f2174886f7 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -1959,14 +1959,19 @@ def MemRefReinterpretCastOp:
   let builders = [
     // Build a ReinterpretCastOp with mixed static and dynamic entries.
     OpBuilderDAG<(ins "MemRefType":$resultType, "Value":$source,
-      "int64_t":$staticOffset, "ArrayRef<int64_t>":$staticSizes,
-      "ArrayRef<int64_t>":$staticStrides, "ValueRange":$offset,
-      "ValueRange":$sizes, "ValueRange":$strides,
+      "OpFoldResult":$offset, "ArrayRef<OpFoldResult>":$sizes,
+      "ArrayRef<OpFoldResult>":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a ReinterpretCastOp with all dynamic entries.
+    // Build a ReinterpretCastOp with static entries.
     OpBuilderDAG<(ins "MemRefType":$resultType, "Value":$source,
-      "Value":$offset, "ValueRange":$sizes, "ValueRange":$strides,
+      "int64_t":$offset, "ArrayRef<int64_t>":$sizes,
+      "ArrayRef<int64_t>":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
+    // Build a ReinterpretCastOp with dynamic entries.
+    OpBuilderDAG<(ins "MemRefType":$resultType, "Value":$source,
+      "Value":$offset, "ValueRange":$sizes,
+      "ValueRange":$strides,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>
   ];
 
   let extraClassDeclaration = extraBaseClassDeclaration # [{
@@ -2927,23 +2932,33 @@ def SubViewOp : BaseOpWithOffsetSizesAndStrides<
   let results = (outs AnyMemRef:$result);
 
   let builders = [
-    // Build a SubViewOp with mixed static and dynamic entries.
-    OpBuilderDAG<(ins "Value":$source, "ArrayRef<int64_t>":$staticOffsets,
-      "ArrayRef<int64_t>":$staticSizes, "ArrayRef<int64_t>":$staticStrides,
-      "ValueRange":$offsets, "ValueRange":$sizes, "ValueRange":$strides,
+    // Build a SubViewOp with mixed static and dynamic entries and custom
+    // result type. If the type passed is nullptr, it is inferred.
+    OpBuilderDAG<(ins "Value":$source, "ArrayRef<OpFoldResult>":$offsets,
+      "ArrayRef<OpFoldResult>":$sizes, "ArrayRef<OpFoldResult>":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubViewOp with all dynamic entries.
-    OpBuilderDAG<(ins "Value":$source, "ValueRange":$offsets,
-      "ValueRange":$sizes, "ValueRange":$strides,
+    // Build a SubViewOp with mixed static and dynamic entries and inferred
+    // result type.
+    OpBuilderDAG<(ins "MemRefType":$resultType, "Value":$source,
+      "ArrayRef<OpFoldResult>":$offsets, "ArrayRef<OpFoldResult>":$sizes,
+      "ArrayRef<OpFoldResult>":$strides,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
+    // Build a SubViewOp with static entries and custom result type. If the
+    // type passed is nullptr, it is inferred.
+    OpBuilderDAG<(ins "Value":$source, "ArrayRef<int64_t>":$offsets,
+      "ArrayRef<int64_t>":$sizes, "ArrayRef<int64_t>":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubViewOp with mixed static and dynamic entries
-    // and custom result type.
+    // Build a SubViewOp with static entries and inferred result type.
     OpBuilderDAG<(ins "MemRefType":$resultType, "Value":$source,
-      "ArrayRef<int64_t>":$staticOffsets, "ArrayRef<int64_t>":$staticSizes,
-      "ArrayRef<int64_t>":$staticStrides, "ValueRange":$offsets,
+      "ArrayRef<int64_t>":$offsets, "ArrayRef<int64_t>":$sizes,
+      "ArrayRef<int64_t>":$strides,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
+    // Build a SubViewOp with dynamic entries and custom result type. If the
+    // type passed is nullptr, it is inferred.
+    OpBuilderDAG<(ins "Value":$source, "ValueRange":$offsets,
       "ValueRange":$sizes, "ValueRange":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubViewOp with all dynamic entries and custom result type.
+    // Build a SubViewOp with dynamic entries and inferred result type.
     OpBuilderDAG<(ins "MemRefType":$resultType, "Value":$source,
       "ValueRange":$offsets, "ValueRange":$sizes, "ValueRange":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>
@@ -3039,23 +3054,23 @@ def SubTensorOp : BaseOpWithOffsetSizesAndStrides<
   let results = (outs AnyRankedTensor:$result);
 
   let builders = [
-    // Build a SubTensorOp with mixed static and dynamic entries.
-    OpBuilderDAG<(ins "Value":$source, "ArrayRef<int64_t>":$staticOffsets,
-      "ArrayRef<int64_t>":$staticSizes, "ArrayRef<int64_t>":$staticStrides,
-      "ValueRange":$offsets, "ValueRange":$sizes, "ValueRange":$strides,
+    // Build a SubTensorOp with mixed static and dynamic entries and inferred
+    // result type.
+    OpBuilderDAG<(ins "Value":$source, "ArrayRef<OpFoldResult>":$offsets,
+      "ArrayRef<OpFoldResult>":$sizes, "ArrayRef<OpFoldResult>":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubTensorOp with all dynamic entries.
-    OpBuilderDAG<(ins "Value":$source, "ValueRange":$offsets,
-      "ValueRange":$sizes, "ValueRange":$strides,
-      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubTensorOp with mixed static and dynamic entries
-    // and custom result type.
+    // Build a SubTensorOp with mixed static and dynamic entries and custom
+    // result type. If the type passed is nullptr, it is inferred.
     OpBuilderDAG<(ins "RankedTensorType":$resultType, "Value":$source,
-      "ArrayRef<int64_t>":$staticOffsets, "ArrayRef<int64_t>":$staticSizes,
-      "ArrayRef<int64_t>":$staticStrides, "ValueRange":$offsets,
+      "ArrayRef<OpFoldResult>":$offsets, "ArrayRef<OpFoldResult>":$sizes,
+      "ArrayRef<OpFoldResult>":$strides,
+      CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
+    // Build a SubTensorOp with dynamic entries and custom result type. If the
+    // type passed is nullptr, it is inferred.
+    OpBuilderDAG<(ins "Value":$source, "ValueRange":$offsets,
       "ValueRange":$sizes, "ValueRange":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubTensorOp with all dynamic entries and custom result type.
+    // Build a SubTensorOp with dynamic entries and inferred result type.
     OpBuilderDAG<(ins "RankedTensorType":$resultType, "Value":$source,
       "ValueRange":$offsets, "ValueRange":$sizes, "ValueRange":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>
@@ -3147,13 +3162,12 @@ def SubTensorInsertOp : BaseOpWithOffsetSizesAndStrides<
   let builders = [
     // Build a SubTensorInsertOp with mixed static and dynamic entries.
     OpBuilderDAG<(ins "Value":$source, "Value":$dest,
-      "ArrayRef<int64_t>":$staticOffsets, "ArrayRef<int64_t>":$staticSizes,
-      "ArrayRef<int64_t>":$staticStrides, "ValueRange":$offsets,
-      "ValueRange":$sizes, "ValueRange":$strides,
+      "ArrayRef<OpFoldResult>":$offsets, "ArrayRef<OpFoldResult>":$sizes,
+      "ArrayRef<OpFoldResult>":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>,
-    // Build a SubTensorInsertOp with all dynamic entries.
-    OpBuilderDAG<(ins "Value":$source, "Value":$dest, "ValueRange":$offsets,
-      "ValueRange":$sizes, "ValueRange":$strides,
+    // Build a SubTensorInsertOp with dynamic entries.
+    OpBuilderDAG<(ins "Value":$source, "Value":$dest,
+      "ValueRange":$offsets, "ValueRange":$sizes, "ValueRange":$strides,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>
   ];
 
diff --git a/mlir/include/mlir/IR/ImplicitLocOpBuilder.h b/mlir/include/mlir/IR/ImplicitLocOpBuilder.h
index ff85f2b5a382..862be32e6208 100644
--- a/mlir/include/mlir/IR/ImplicitLocOpBuilder.h
+++ b/mlir/include/mlir/IR/ImplicitLocOpBuilder.h
@@ -22,20 +22,11 @@ namespace mlir {
 /// as OpBuilder.
 class ImplicitLocOpBuilder : public mlir::OpBuilder {
 public:
-  /// Create an ImplicitLocOpBuilder using the insertion point and listener from
-  /// an existing OpBuilder.
-  ImplicitLocOpBuilder(Location loc, const OpBuilder &builder)
-      : OpBuilder(builder), curLoc(loc) {}
-
   /// OpBuilder has a bunch of convenience constructors - we support them all
   /// with the additional Location.
-  template <typename T>
-  ImplicitLocOpBuilder(Location loc, T &&operand, Listener *listener = nullptr)
-      : OpBuilder(std::forward<T>(operand), listener), curLoc(loc) {}
-
-  ImplicitLocOpBuilder(Location loc, Block *block, Block::iterator insertPoint,
-                       Listener *listener = nullptr)
-      : OpBuilder(block, insertPoint, listener), curLoc(loc) {}
+  template <typename... T>
+  ImplicitLocOpBuilder(Location loc, T &&...operands)
+      : OpBuilder(std::forward<T>(operands)...), curLoc(loc) {}
 
   /// Create a builder and set the insertion point to before the first operation
   /// in the block but still inside the block.
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 1180938e858b..3b5e661fdef7 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -213,11 +213,25 @@ inline bool operator!=(OpState lhs, OpState rhs) {
   return lhs.getOperation() != rhs.getOperation();
 }
 
+raw_ostream &operator<<(raw_ostream &os, OpFoldResult ofr);
+
 /// This class represents a single result from folding an operation.
 class OpFoldResult : public PointerUnion<Attribute, Value> {
   using PointerUnion<Attribute, Value>::PointerUnion;
+
+public:
+  void dump() { llvm::errs() << *this << "\n"; }
 };
 
+/// Allow printing to a stream.
+inline raw_ostream &operator<<(raw_ostream &os, OpFoldResult ofr) {
+  if (Value value = ofr.dyn_cast<Value>())
+    value.print(os);
+  else
+    ofr.dyn_cast<Attribute>().print(os);
+  return os;
+}
+
 /// Allow printing to a stream.
 inline raw_ostream &operator<<(raw_ostream &os, OpState &op) {
   op.print(os, OpPrintingFlags().useLocalScope());
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.h b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
index d5f44c3e63da..2b3a054338ab 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.h
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
@@ -14,6 +14,7 @@
 #define MLIR_INTERFACES_VIEWLIKEINTERFACE_H_
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
 
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.td b/mlir/include/mlir/Interfaces/ViewLikeInterface.td
index 62c371b2f97d..23b03a26d7ad 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.td
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.td
@@ -156,6 +156,72 @@ def OffsetSizeAndStrideOpInterface : OpInterface<"OffsetSizeAndStrideOpInterface
         return $_op.static_strides();
       }]
     >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return a vector of all the static or dynamic sizes of the op.
+      }],
+      /*retTy=*/"SmallVector<OpFoldResult, 4>",
+      /*methodName=*/"getMixedOffsets",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        SmallVector<OpFoldResult, 4> res;
+        std::array<unsigned, 3> ranks = $_op.getArrayAttrRanks();
+        unsigned numDynamic = 0;
+        unsigned count = ranks[getOffsetOperandGroupPosition()];
+        for (unsigned idx = 0; idx < count; ++idx) {
+          if (isDynamicOffset(idx))
+            res.push_back($_op.offsets()[numDynamic++]);
+          else
+            res.push_back($_op.static_offsets()[idx]);
+        }
+        return res;
+      }]
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return a vector of all the static or dynamic sizes of the op.
+      }],
+      /*retTy=*/"SmallVector<OpFoldResult, 4>",
+      /*methodName=*/"getMixedSizes",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        SmallVector<OpFoldResult, 4> res;
+        std::array<unsigned, 3> ranks = $_op.getArrayAttrRanks();
+        unsigned numDynamic = 0;
+        unsigned count = ranks[getSizeOperandGroupPosition()];
+        for (unsigned idx = 0; idx < count; ++idx) {
+          if (isDynamicSize(idx))
+            res.push_back($_op.sizes()[numDynamic++]);
+          else
+            res.push_back($_op.static_sizes()[idx]);
+        }
+        return res;
+      }]
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return a vector of all the static or dynamic strides of the op.
+      }],
+      /*retTy=*/"SmallVector<OpFoldResult, 4>",
+      /*methodName=*/"getMixedStrides",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        SmallVector<OpFoldResult, 4> res;
+        std::array<unsigned, 3> ranks = $_op.getArrayAttrRanks();
+        unsigned numDynamic = 0;
+        unsigned count = ranks[getStrideOperandGroupPosition()];
+        for (unsigned idx = 0; idx < count; ++idx) {
+          if (isDynamicStride(idx))
+            res.push_back($_op.strides()[numDynamic++]);
+          else
+            res.push_back($_op.static_strides()[idx]);
+        }
+        return res;
+      }]
+    >,
 
     InterfaceMethod<
       /*desc=*/[{
@@ -359,6 +425,9 @@ def OffsetSizeAndStrideOpInterface : OpInterface<"OffsetSizeAndStrideOpInterface
   ];
 
   let extraClassDeclaration = [{
+    static unsigned getOffsetOperandGroupPosition() { return 0; }
+    static unsigned getSizeOperandGroupPosition() { return 1; }
+    static unsigned getStrideOperandGroupPosition() { return 2; }
     static StringRef getStaticOffsetsAttrName() {
       return "static_offsets";
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
index 5df7fe982281..f663539628c5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
@@ -241,10 +241,8 @@ class SubTensorOpConverter : public OpConversionPattern<SubTensorOp> {
     Value alloc =
         rewriter.create<AllocOp>(op.getLoc(), subviewMemRefType, op.sizes());
     Value subView = rewriter.create<SubViewOp>(
-        op.getLoc(), sourceMemref, extractFromI64ArrayAttr(op.static_offsets()),
-        extractFromI64ArrayAttr(op.static_sizes()),
-        extractFromI64ArrayAttr(op.static_strides()), op.offsets(), op.sizes(),
-        op.strides());
+        op.getLoc(), sourceMemref, op.getMixedOffsets(), op.getMixedSizes(),
+        op.getMixedStrides());
     rewriter.create<linalg::CopyOp>(op.getLoc(), subView, alloc);
     rewriter.replaceOp(op, alloc);
     return success();
@@ -283,10 +281,8 @@ class SubTensorInsertOpConverter
 
     // Take a subview to copy the small memref.
     Value subview = rewriter.create<SubViewOp>(
-        op.getLoc(), destMemRef, extractFromI64ArrayAttr(op.static_offsets()),
-        extractFromI64ArrayAttr(op.static_sizes()),
-        extractFromI64ArrayAttr(op.static_strides()), adaptor.offsets(),
-        adaptor.sizes(), adaptor.strides());
+        op.getLoc(), destMemRef, op.getMixedOffsets(), op.getMixedSizes(),
+        op.getMixedStrides());
     // Copy the small memref.
     rewriter.create<linalg::CopyOp>(op.getLoc(), sourceMemRef, subview);
     rewriter.replaceOp(op, destMemRef);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 714bb0f97777..4fe90897873b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -61,9 +61,9 @@ using llvm::dbgs;
 // by `permutationMap`.
 static void inferShapeComponents(AffineMap permutationMap,
                                  ArrayRef<Range> loopRanges,
-                                 SmallVectorImpl<Value> &offsets,
-                                 SmallVectorImpl<Value> &sizes,
-                                 SmallVectorImpl<Value> &strides) {
+                                 SmallVectorImpl<OpFoldResult> &offsets,
+                                 SmallVectorImpl<OpFoldResult> &sizes,
+                                 SmallVectorImpl<OpFoldResult> &strides) {
   assert(permutationMap.isProjectedPermutation() &&
          "expected some subset of a permutation map");
   SmallVector<Range, 4> shapeRanges(permutationMap.getNumResults());
@@ -101,7 +101,7 @@ static LinalgOp cloneWithLoopRanges(OpBuilder &b, Location loc, LinalgOp op,
     AffineMap map = op.getIndexingMap(shapedOperandIdx);
     LLVM_DEBUG(llvm::dbgs() << "shapedOperandIdx: " << shapedOperandIdx
                             << " with indexingMap: " << map << "\n");
-    SmallVector<Value, 4> offsets, sizes, strides;
+    SmallVector<OpFoldResult, 4> offsets, sizes, strides;
     inferShapeComponents(map, loopRanges, offsets, sizes, strides);
     Value shape = en.value();
     Value sub = shape.getType().isa<MemRefType>()
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 98d61fa6a8d9..5c67c8e61829 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -334,3 +334,275 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
     });
   }
 }
+
+/// Ensure prerequisites that guarantee pad op hoisting can occur.
+/// Return failure in the cases when we cannot perform hoisting; i.e. if either:
+///   1. There exists a use of `simplePadOp` that is not a linalg input operand.
+///   2. There isn't an enclosing `outermostEnclosingForOp` loop.
+///   3. There exists an op with a region that is dominated by
+///   `outermostEnclosingForOp` and that isn't a LoopLikeInterface or a
+///    LinalgOp.
+///   3. There exists an op with side effects that is dominated by
+///    `outermostEnclosingForOp` and that isn't a LoopLikeInterface.
+///
+/// While ensuring prerequisites:
+///   1. Fill the `backwardSlice` to contain the topologically sorted ops
+///   dominated by `outermostEnclosingForOp`.
+///   2. Fill the `packingLoops` to contain only the enclosing loops of
+///   `backwardSlice` whose IV is actually used in computing padding. Loops that
+///   remain in `backwardSlice` but that are not in `packingLoops` are
+///   dimensions of reuse.
+static LogicalResult
+hoistPaddingOnTensorsPrerequisites(linalg::SimplePadOp simplePadOp, int nLevels,
+                                   llvm::SetVector<Operation *> &backwardSlice,
+                                   llvm::SetVector<Operation *> &packingLoops) {
+  // Bail on any use that isn't an input of a Linalg op.
+  // Hoisting of inplace updates happens after vectorization.
+  for (OpOperand &use : simplePadOp.result().getUses()) {
+    auto linalgUser = dyn_cast<linalg::LinalgOp>(use.getOwner());
+    if (!linalgUser || !linalgUser.isInputTensor(&use))
+      return failure();
+  }
+
+  // Get at most nLevels of enclosing loops.
+  SmallVector<LoopLikeOpInterface> reverseEnclosingLoops;
+  Operation *outermostEnclosingForOp = nullptr,
+            *nextEnclosingForOp =
+                simplePadOp->getParentOfType<LoopLikeOpInterface>();
+  while (nLevels-- > 0 && nextEnclosingForOp) {
+    outermostEnclosingForOp = nextEnclosingForOp;
+    reverseEnclosingLoops.push_back(outermostEnclosingForOp);
+    nextEnclosingForOp =
+        nextEnclosingForOp->getParentOfType<LoopLikeOpInterface>();
+  }
+  if (!outermostEnclosingForOp)
+    return failure();
+
+  // Get the backwards slice from `simplePadOp` that is dominated by the
+  // outermost enclosing loop.
+  DominanceInfo domInfo(outermostEnclosingForOp);
+  getBackwardSlice(simplePadOp, &backwardSlice, [&](Operation *op) {
+    return domInfo.dominates(outermostEnclosingForOp, op);
+  });
+
+  #if 0
+
+  // Bail on any op with a region that is not a LoopLikeInterface or a LinalgOp.
+  // Bail on any op with side effects that is not a LoopLikeInterface.
+  if (llvm::any_of(backwardSlice, [](Operation *op) {
+        if (isa<LoopLikeOpInterface>(op))
+          return false;
+        if (!MemoryEffectOpInterface::hasNoEffect(op))
+          return true;
+        return op->getNumRegions() > 0 && !isa<LinalgOp>(op);
+      }))
+    return failure();
+
+  #else
+
+  // Bail on any op with a region that is not a LoopLikeInterface or a LinalgOp.
+  if (llvm::any_of(backwardSlice, [](Operation *op) {
+        return op->getNumRegions() > 0 && !isa<LoopLikeOpInterface>(op) &&
+               !isa<LinalgOp>(op);
+      }))
+    return failure();
+
+  #endif
+
+  // Filter out the loops whose induction variable is not used to compute the
+  // padded result. As a first approximation, just look for IVs that have no use
+  // in the backwardSlice.
+  // These are the dimensions of reuse that we can exploit to reduce the amount
+  // of work / memory.
+  // TODO: would this optimization compose better as a canonicalization?
+  for (LoopLikeOpInterface loop : reverseEnclosingLoops) {
+    auto forOp = dyn_cast<scf::ForOp>(loop.getOperation());
+    if (!forOp)
+      continue;
+    for (Operation *user : forOp.getInductionVar().getUsers()) {
+      if (backwardSlice.contains(user)) {
+        packingLoops.insert(forOp);
+        break;
+      }
+    }
+  }
+
+  // Backward slice is a topologically sorted list of ops starting at
+  // `outermostEnclosingForOp`.
+  assert(outermostEnclosingForOp == backwardSlice.front());
+
+  return success();
+}
+
+static Value buildLoopTripCount(OpBuilder &b, Operation *op) {
+  MLIRContext *ctx = op->getContext();
+  AffineExpr lb, ub, step = getAffineSymbolExpr(0, ctx);
+  bindDims(ctx, lb, ub);
+  scf::ForOp forOp = cast<scf::ForOp>(op);
+  return b.create<AffineApplyOp>(
+      op->getLoc(), AffineMap::get(2, 1, {(ub - lb).ceilDiv(step)}, ctx),
+      ValueRange{forOp.lowerBound(), forOp.upperBound(), forOp.step()});
+}
+
+/// Mechanically hoist padding operations on tensors by at most `nLoops` into a
+/// new, generally larger tensor. This achieves packing of multiple padding ops
+/// into a larger tensor. On success, `simplePadOp` is replaced by the cloned
+/// version in the packing loop so the caller can continue reasoning about the
+/// padding operation.
+///
+/// Example in pseudo-mlir:
+/// =======================
+///
+/// If hoistPaddingOnTensors is called with `nLoops` = 2 on the following IR.
+/// ```
+///    scf.for (%i, %j, %k)
+///      %st0 = subtensor f(%i, %k) : ... to tensor<?x?xf32>
+///      %0 = linalg.simple_pad %st0 pad %pad :
+///             tensor<?x?xf32> to tensor<4x8xf32>
+///      compute(%0)
+/// ```
+///
+/// IR resembling the following is produced:
+///
+/// ```
+///    scf.for (%i) {
+///      %packed_init = linalg.init_tensor range(%j) : tensor<?x4x8xf32>
+///      %packed = scf.for (%k) iter_args(%p : %packed_init)
+///        %st0 = subtensor f(%i, %k) : ... to tensor<?x?xf32>
+///        %0 = linalg.simple_pad %st0 pad %pad :
+///               tensor<?x?xf32> to tensor<4x8xf32>
+///        scf.yield %1: tensor<?x4x8xf32>
+///      } -> tensor<?x4x8xf32>
+///      scf.for (%j, %k) {
+///        %st0 = subtensor %packed [%k, 0, 0][1, 4, 8][1, 1, 1] :
+///                 tensor<?x4x8xf32> to tensor<4x8xf32>
+///        compute(%st0)
+///      }
+///    }
+/// ```
+LogicalResult mlir::linalg::hoistPaddingOnTensors(SimplePadOp &simplePadOp,
+                                                  unsigned nLoops) {
+  llvm::SetVector<Operation *> backwardSlice, packingLoops;
+  if (failed(hoistPaddingOnTensorsPrerequisites(simplePadOp, nLoops,
+                                                backwardSlice, packingLoops)))
+    return failure();
+
+  // Update actual number of loops, which may be smaller.
+  nLoops = packingLoops.size();
+
+  Location loc = simplePadOp->getLoc();
+  RankedTensorType paddedTensorType = simplePadOp.getResultType();
+  unsigned paddedRank = paddedTensorType.getRank();
+
+  // Backward slice is a topologically sorted list of ops starting at
+  // `outermostEnclosingForOp`.
+  Operation *outermostEnclosingForOp = backwardSlice.front();
+  // IP just before the outermost loop considered that we hoist above.
+  OpBuilder b(outermostEnclosingForOp);
+
+  // Create the packed tensor<?x?x..?xpadded_shape> into which we amortize
+  // padding.
+  SmallVector<int64_t> packedShape(nLoops, ShapedType::kDynamicSize);
+  // TODO: go grab dims when necessary, for now SimplePadOp returns a static
+  // tensor.
+  llvm::append_range(packedShape, paddedTensorType.getShape());
+  auto packedTensorType =
+      RankedTensorType::get(packedShape, paddedTensorType.getElementType());
+  auto dynamicSizes = llvm::to_vector<4>(llvm::map_range(
+      packingLoops, [&](Operation *op) { return buildLoopTripCount(b, op); }));
+  Value packedTensor = b.create<linalg::InitTensorOp>(
+      loc, dynamicSizes, packedTensorType.getShape(),
+      packedTensorType.getElementType());
+
+  // Clone the operations involved in the backward slice, iteratively stepping
+  // into the loops that we encounter.
+  // The implementation proceeds in a stack-like fashion:
+  //   1. Iteratively clone and step into the loops, pushing the `packedTensor`
+  //      deeper in the stack.
+  //   2. Create a SubTensorInsert at the top of the stack.
+  //   3. Iteratively pop and yield the result of the SubTensorInsertOp across
+  //     the cloned loops.
+  SmallVector<Value> clonedLoopIvs;
+  clonedLoopIvs.reserve(nLoops);
+  BlockAndValueMapping bvm;
+  // Stack step 1. iteratively clone loops and push `packedTensor`.
+  // Insert `simplePadOp` into the backwardSlice so we clone it too.
+  backwardSlice.insert(simplePadOp);
+  for (Operation *op : backwardSlice) {
+    if (op->getNumRegions() == 0) {
+      b.clone(*op, bvm);
+      continue;
+    }
+    // TODO: support more cases as they appear.
+    auto forOp = dyn_cast<scf::ForOp>(op);
+    assert(forOp && "Expected scf::ForOp when hoisting pad ops");
+    // Unused loop, just skip it.
+    if (!packingLoops.contains(forOp))
+      continue;
+    auto clonedForOp =
+        b.create<scf::ForOp>(loc, forOp.lowerBound(), forOp.upperBound(),
+                             forOp.step(), packedTensor);
+    assert(clonedForOp->getNumRegions() == 1);
+    clonedLoopIvs.push_back(clonedForOp.getInductionVar());
+    b.setInsertionPointToStart(&clonedForOp->getRegion(0).front());
+    bvm.map(forOp.getInductionVar(), clonedLoopIvs.back());
+    packedTensor = clonedForOp.getRegionIterArgs().front();
+  }
+
+  // Stack step 2. create SubTensorInsertOp at the top of the stack.
+  // offsets = [clonedLoopIvs, 0 .. 0].
+  SmallVector<OpFoldResult> offsets(clonedLoopIvs.begin(), clonedLoopIvs.end());
+  offsets.append(paddedRank, b.getIndexAttr(0));
+  // sizes = [1 .. 1, paddedShape].
+  SmallVector<OpFoldResult> sizes(nLoops, b.getIndexAttr(1));
+  for (int64_t sz : paddedTensorType.getShape()) {
+    // TODO: go grab dims when necessary, for now SimplePadOp returns a static
+    // tensor.
+    assert(!ShapedType::isDynamic(sz) && "padded tensor needs static sizes");
+    sizes.push_back(b.getIndexAttr(sz));
+  }
+  // strides = [1 .. 1].
+  SmallVector<OpFoldResult> strides(nLoops + paddedRank, b.getIndexAttr(1));
+
+  Value inserted =
+      b.create<SubTensorInsertOp>(loc, bvm.lookup(simplePadOp.result()),
+                                  packedTensor, offsets, sizes, strides);
+
+  // Stack step 3. iteratively pop the stack and propagate the yield.
+  Value valueToYield = inserted;
+  for (Value iv : llvm::reverse(clonedLoopIvs)) {
+    auto forOp = scf::getForInductionVarOwner(iv);
+    b.setInsertionPointToEnd(&forOp.getRegion().front());
+    b.create<scf::YieldOp>(loc, valueToYield);
+    valueToYield = forOp.getResult(0);
+  }
+
+  // Now the packed tensor is ready, replace the original padding op by a
+  // 1x..x1 SubTensor [originalLoopIvs, 0 .. 0][1 .. 1, paddedShape][1 .. 1].
+  b.setInsertionPoint(simplePadOp);
+  SmallVector<Value> originalLoopIvs =
+      llvm::to_vector<4>(llvm::map_range(packingLoops, [](Operation *loop) {
+        return cast<scf::ForOp>(loop).getInductionVar();
+      }));
+  // offsets = [originalLoopIvs, 0 .. 0].
+  offsets.assign(originalLoopIvs.begin(), originalLoopIvs.end());
+  offsets.append(paddedRank, b.getIndexAttr(0));
+  // sizes = [1 .. 1, paddedShape] (definedabove).
+  // strides = [1 .. 1] (defined above)
+  packedTensor =
+      scf::getForInductionVarOwner(clonedLoopIvs.front())->getResult(0);
+  simplePadOp.replaceAllUsesWith(
+      b.create<SubTensorOp>(loc, simplePadOp.getResultType(), packedTensor,
+                            offsets, sizes, strides)
+          ->getResult(0));
+
+  Operation *toErase = simplePadOp;
+
+  // Make the newly cloned `simplePadOp` available to the caller.
+  simplePadOp =
+      cast<SimplePadOp>(bvm.lookup(simplePadOp.result()).getDefiningOp());
+
+  toErase->erase();
+
+  return success();
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index 4d314c32657a..5b01dd04b55f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -214,7 +214,8 @@ Optional<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
   ScopedContext scopedContext(b, loc);
   auto viewType = subView.getType();
   auto rank = viewType.getRank();
-  SmallVector<Value, 4> fullSizes, partialSizes;
+  SmallVector<Value, 4> fullSizes;
+  SmallVector<OpFoldResult> partialSizes;
   fullSizes.reserve(rank);
   partialSizes.reserve(rank);
   for (auto en : llvm::enumerate(subView.getOrCreateRanges(b, loc))) {
@@ -226,7 +227,7 @@ Optional<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
         (!sizeAttr) ? rangeValue.size : b.create<ConstantOp>(loc, sizeAttr);
     LLVM_DEBUG(llvm::dbgs() << "Extracted tightest: " << size << "\n");
     fullSizes.push_back(size);
-    partialSizes.push_back(folded_std_dim(folder, subView, en.index()));
+    partialSizes.push_back(folded_std_dim(folder, subView, en.index()).value);
   }
   SmallVector<int64_t, 4> dynSizes(fullSizes.size(), -1);
   // If a callback is not specified, then use the default implementation for
@@ -234,10 +235,8 @@ Optional<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
   Optional<Value> fullLocalView = allocationFn(b, subView, fullSizes, folder);
   if (!fullLocalView)
     return {};
-  auto zero = folded_std_constant_index(folder, 0);
-  auto one = folded_std_constant_index(folder, 1);
-  SmallVector<Value, 4> zeros(fullSizes.size(), zero);
-  SmallVector<Value, 4> ones(fullSizes.size(), one);
+  SmallVector<OpFoldResult, 4> zeros(fullSizes.size(), b.getIndexAttr(0));
+  SmallVector<OpFoldResult, 4> ones(fullSizes.size(), b.getIndexAttr(1));
   auto partialLocalView =
       folded_std_subview(folder, *fullLocalView, zeros, partialSizes, ones);
   return PromotionInfo{*fullLocalView, partialLocalView};
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 62a5d325ddcf..d723dc47ac57 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -255,15 +255,15 @@ makeTiledShapes(OpBuilder &b, Location loc, LinalgOp linalgOp,
     }
 
     // Construct a new subview / subtensor for the tile.
-    SmallVector<Value, 4> offsets, sizes, strides;
+    SmallVector<OpFoldResult, 4> offsets, sizes, strides;
     offsets.reserve(rank);
     sizes.reserve(rank);
     strides.reserve(rank);
     for (unsigned r = 0; r < rank; ++r) {
       if (!isTiled(map.getSubMap({r}), tileSizes)) {
-        offsets.push_back(std_constant_index(0));
-        sizes.push_back(std_dim(shapedOp, r));
-        strides.push_back(std_constant_index(1));
+        offsets.push_back(b.getIndexAttr(0));
+        sizes.push_back(std_dim(shapedOp, r).value);
+        strides.push_back(b.getIndexAttr(1));
         continue;
       }
 
@@ -297,7 +297,7 @@ makeTiledShapes(OpBuilder &b, Location loc, LinalgOp linalgOp,
       }
 
       sizes.push_back(size);
-      strides.push_back(std_constant_index(1));
+      strides.push_back(b.getIndexAttr(1));
     }
 
     if (shapedType.isa<MemRefType>())
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 283ff20f611b..b0cb51516e25 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -25,6 +25,7 @@
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include <type_traits>
@@ -105,6 +106,116 @@ mlir::linalg::LinalgTilingOptions::setTileSizes(ArrayRef<int64_t> ts) {
   return *this;
 }
 
+/// Try to compute a static bounding box for `operand`
+/// Return success if either:
+///   1. The operand is already statically shaped, `result` is left unchanged.
+///   2. The operand is (partially) dynamic, `result` is the result of a freshly
+///      created SimplePadOp.
+/// Return failure if the operand cannot be padded to a static shape.
+static LogicalResult padOperandToSmallestStaticBoundingBox(
+    PatternRewriter &rewriter, linalg::LinalgOp opToPad, Value operand,
+    const LinalgTilingOptions &options, Value &result) {
+  auto tensorType = operand.getType().cast<RankedTensorType>();
+  // Already static shape, no need to pad.
+  if (tensorType.hasStaticShape())
+    return success();
+  auto subtensor = operand.getDefiningOp<SubTensorOp>();
+  // Not a subtensor, cannot construct a static bounding box.
+  if (!subtensor)
+    return failure();
+  SmallVector<int64_t> staticSizes;
+  staticSizes.reserve(tensorType.getRank());
+  auto shapedOp =
+      cast<OffsetSizeAndStrideOpInterface>(subtensor.getOperation());
+  for (auto size : shapedOp.getMixedSizes()) {
+    auto indexAttr = size.is<Attribute>()
+                         ? size.get<Attribute>().dyn_cast<IntegerAttr>()
+                         : linalg::getSmallestBoundingIndex(size.get<Value>());
+    // SmallestBoundingIndex must exist for all sizes.
+    // For now return an error if we can't find it.
+    if (!indexAttr)
+      return rewriter.notifyMatchFailure(
+          opToPad, "No constant bounding box can be found for padding");
+    staticSizes.push_back(indexAttr.getInt());
+  }
+  Value pad = options.paddingValueComputationFunction(rewriter, opToPad);
+  auto staticTensorType =
+      RankedTensorType::get(staticSizes, tensorType.getElementType());
+  result = rewriter.create<linalg::SimplePadOp>(opToPad->getLoc(),
+                                                staticTensorType, operand, pad);
+  return success();
+}
+
+// Try to create a static bounding box around each operand of `res.op`.
+// If successful, `res.op` is rewritten in static form with padded operands.
+// `res.op` is updated to the cloned static form of the op on success.
+static LogicalResult rewriteAsPaddedOp(PatternRewriter &rewriter,
+                                       TiledLinalgOp &res,
+                                       const LinalgTilingOptions &options) {
+  LinalgOp opToPad = res.op;
+  Location loc = opToPad->getLoc();
+
+  // If the op is fully static, it does not need padding.
+  // TODO: there are cases where we may still want to pad to larger sizes.
+  if (llvm::all_of(opToPad.getShapedOperands(), [](Value v) {
+        return v.getType().cast<RankedTensorType>().hasStaticShape();
+      }))
+    return success();
+
+  OpBuilder::InsertionGuard g(rewriter);
+  // Set IP after op because we also take the dims of the original output.
+  rewriter.setInsertionPointAfter(opToPad);
+  // Make a copy of the shaped operands and update it.
+  SmallVector<Value> operands = opToPad.getShapedOperands();
+  for (Value &v : operands) {
+    Value paddedOperand;
+    // If padding was requested but the shape cannot be bounded statically then
+    // the pattern fails to apply.
+    if (failed(padOperandToSmallestStaticBoundingBox(rewriter, opToPad, v,
+                                                     options, paddedOperand))) {
+      return failure();
+    }
+    // Update v if we indeed got a padded operand.
+    v = paddedOperand ? paddedOperand : v;
+  }
+
+  // Clone `opToPad` to operate on the statically padded shapes.
+  auto resultTensorTypes =
+      ValueRange(operands).take_back(opToPad.getNumOutputs()).getTypes();
+  ValueRange otherOperands = opToPad.getAssumedNonShapedOperands();
+  operands.append(otherOperands.begin(), otherOperands.end());
+  linalg::LinalgOp paddedOp =
+      opToPad.clone(rewriter, loc, resultTensorTypes, operands);
+
+  // Recover the subtensor out of the new static results. This keeps the
+  // original linalg op around because it uses the dims of the original results.
+  // This later folds away.
+  SmallVector<Value> paddedSubviewResults;
+  paddedSubviewResults.reserve(opToPad->getNumResults());
+  llvm::SetVector<Operation *> newUsersOfOpToPad;
+  for (auto it : llvm::zip(opToPad->getResults(), paddedOp->getResults())) {
+    auto rank = std::get<0>(it).getType().cast<RankedTensorType>().getRank();
+    SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
+    auto sizes = llvm::to_vector<4>(llvm::map_range(
+        llvm::seq<unsigned>(0, rank), [&](unsigned d) -> OpFoldResult {
+          auto dimOp = rewriter.create<DimOp>(loc, std::get<0>(it), d);
+          newUsersOfOpToPad.insert(dimOp);
+          return dimOp.getResult();
+        }));
+    SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
+    paddedSubviewResults.push_back(rewriter.create<SubTensorOp>(
+        loc, std::get<1>(it), offsets, sizes, strides));
+  }
+  // Replace the transient `opToPad` locally, except for uses that we just
+  // created for the purpose of extracting the dims.
+  rewriter.replaceOpWithIf(opToPad, paddedSubviewResults, [&](OpOperand &opOp) {
+    return !newUsersOfOpToPad.contains(opOp.getOwner());
+  });
+
+  res = TiledLinalgOp{paddedOp, res.loops, res.tensorResults};
+  return success();
+}
+
 /// Linalg base tiling pattern.
 mlir::linalg::LinalgBaseTilingPattern::LinalgBaseTilingPattern(
     StringRef opName, MLIRContext *context, LinalgTilingOptions options,
@@ -130,11 +241,34 @@ LogicalResult mlir::linalg::LinalgBaseTilingPattern::matchAndRewriteBase(
   if (!res)
     return failure();
 
-  // Return relevant information to derived pattern.
-  result = *res;
+  // Setup RAII guard to return properly.
+  bool succeeded = true;
+  LinalgOp tiledOp = res->op;
+  auto guard = llvm::make_scope_exit([&]() {
+    if (!succeeded)
+      return;
+    // Return relevant information to derived pattern.
+    result = *res;
+    // Replace marker on both tiledOp and tiledAndPaddedOp, if necessary.
+    marker.replaceLinalgMarker(rewriter, tiledOp);
+    if (tiledOp != res->op)
+      marker.replaceLinalgMarker(rewriter, res->op);
+  });
+
+  // Consider padding on the fly only if the op has tensor semantics.
+  if (!options.paddingValueComputationFunction ||
+      !linalgOp.hasTensorSemantics())
+    return success();
+
+  // Try to pad on the fly by rewriting res->op as a padded op.
+  if (failed(rewriteAsPaddedOp(rewriter, *res, options))) {
+    // Set so RAII guard does not propagate TiledLinalgOp to `result`.
+    succeeded = false;
+    return failure();
+  }
 
-  // New marker if specified.
-  marker.replaceLinalgMarker(rewriter, res->op.getOperation());
+  // Do not perform replacement of `linalgOp`, let the derived patterns
+  // do this as they see fit, from the resulting TiledLinalgOp.
   return success();
 }
 
diff --git a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
index 45dd0fd0086a..124380c1ec08 100644
--- a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
+++ b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
@@ -33,6 +33,32 @@
 
 using namespace mlir;
 
+/// Helper function to dispatch an OpFoldResult into either the `dynamicVec` if
+/// it is a Value or into `staticVec` if it is an IntegerAttr.
+/// In the case of a Value, a copy of the `sentinel` value is also pushed to
+/// `staticVec`. This is useful to extract mixed static and dynamic entries that
+/// come from an AttrSizedOperandSegments trait.
+static void dispatchIndexOpFoldResult(OpFoldResult ofr,
+                                      SmallVectorImpl<Value> &dynamicVec,
+                                      SmallVectorImpl<int64_t> &staticVec,
+                                      int64_t sentinel) {
+  if (auto v = ofr.dyn_cast<Value>()) {
+    dynamicVec.push_back(v);
+    staticVec.push_back(sentinel);
+    return;
+  }
+  APInt apInt = ofr.dyn_cast<Attribute>().cast<IntegerAttr>().getValue();
+  staticVec.push_back(apInt.getSExtValue());
+}
+
+static void dispatchIndexOpFoldResults(ArrayRef<OpFoldResult> ofrs,
+                                       SmallVectorImpl<Value> &dynamicVec,
+                                       SmallVectorImpl<int64_t> &staticVec,
+                                       int64_t sentinel) {
+  for (auto ofr : ofrs)
+    dispatchIndexOpFoldResult(ofr, dynamicVec, staticVec, sentinel);
+}
+
 //===----------------------------------------------------------------------===//
 // StandardOpsDialect Interfaces
 //===----------------------------------------------------------------------===//
@@ -1411,13 +1437,20 @@ OpFoldResult DimOp::fold(ArrayRef<Attribute> operands) {
     return Value{*dynExtents};
   }
 
+  // The size at the given index is now known to be a dynamic size.
+  unsigned unsignedIndex = index.getValue().getZExtValue();
+
+  if (auto subtensor = dyn_cast_or_null<SubTensorOp>(definingOp)) {
+    assert(subtensor.isDynamicSize(unsignedIndex) &&
+           "Expected dynamic subtensor size");
+    return subtensor.getDynamicSize(unsignedIndex);
+  }
+
   // Fold dim to the size argument for an `AllocOp`, `ViewOp`, or `SubViewOp`.
   auto memrefType = argTy.dyn_cast<MemRefType>();
   if (!memrefType)
     return {};
 
-  // The size at the given index is now known to be a dynamic size of a memref.
-  unsigned unsignedIndex = index.getValue().getZExtValue();
   if (auto alloc = dyn_cast_or_null<AllocOp>(definingOp))
     return *(alloc.getDynamicSizes().begin() +
              memrefType.getDynamicDimIndex(unsignedIndex));
@@ -2062,32 +2095,57 @@ OpFoldResult MemRefCastOp::fold(ArrayRef<Attribute> operands) {
 // MemRefReinterpretCastOp
 //===----------------------------------------------------------------------===//
 
-void mlir::MemRefReinterpretCastOp::build(
-    OpBuilder &b, OperationState &result, MemRefType resultType, Value source,
-    int64_t staticOffset, ArrayRef<int64_t> staticSizes,
-    ArrayRef<int64_t> staticStrides, ValueRange offset, ValueRange sizes,
-    ValueRange strides, ArrayRef<NamedAttribute> attrs) {
-  build(b, result, resultType, source, offset, sizes, strides,
-        b.getI64ArrayAttr(staticOffset), b.getI64ArrayAttr(staticSizes),
-        b.getI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
-}
-
 /// Build a MemRefReinterpretCastOp with all dynamic entries: `staticOffsets`,
 /// `staticSizes` and `staticStrides` are automatically filled with
 /// source-memref-rank sentinel values that encode dynamic entries.
+void mlir::MemRefReinterpretCastOp::build(OpBuilder &b, OperationState &result,
+                                          MemRefType resultType, Value source,
+                                          OpFoldResult offset,
+                                          ArrayRef<OpFoldResult> sizes,
+                                          ArrayRef<OpFoldResult> strides,
+                                          ArrayRef<NamedAttribute> attrs) {
+  SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
+  SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
+  dispatchIndexOpFoldResults(offset, dynamicOffsets, staticOffsets,
+                             ShapedType::kDynamicStrideOrOffset);
+  dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes,
+                             ShapedType::kDynamicSize);
+  dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides,
+                             ShapedType::kDynamicStrideOrOffset);
+  build(b, result, resultType, source, dynamicOffsets, dynamicSizes,
+        dynamicStrides, b.getI64ArrayAttr(staticOffsets),
+        b.getI64ArrayAttr(staticSizes), b.getI64ArrayAttr(staticStrides));
+  result.addAttributes(attrs);
+}
+
+void mlir::MemRefReinterpretCastOp::build(OpBuilder &b, OperationState &result,
+                                          MemRefType resultType, Value source,
+                                          int64_t offset,
+                                          ArrayRef<int64_t> sizes,
+                                          ArrayRef<int64_t> strides,
+                                          ArrayRef<NamedAttribute> attrs) {
+  SmallVector<OpFoldResult> sizeValues =
+      llvm::to_vector<4>(llvm::map_range(sizes, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  build(b, result, resultType, source, b.getI64IntegerAttr(offset), sizeValues,
+        strideValues, attrs);
+}
+
 void mlir::MemRefReinterpretCastOp::build(OpBuilder &b, OperationState &result,
                                           MemRefType resultType, Value source,
                                           Value offset, ValueRange sizes,
                                           ValueRange strides,
                                           ArrayRef<NamedAttribute> attrs) {
-  unsigned rank = resultType.getRank();
-  SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
-  SmallVector<int64_t, 4> staticStridesVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  build(b, result, resultType, source,
-        /*staticOffset=*/ShapedType::kDynamicStrideOrOffset, staticSizesVector,
-        staticStridesVector, offset, sizes, strides, attrs);
+  SmallVector<OpFoldResult> sizeValues = llvm::to_vector<4>(
+      llvm::map_range(sizes, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [](Value v) -> OpFoldResult { return v; }));
+  build(b, result, resultType, source, offset, sizeValues, strideValues, attrs);
 }
 
 /// Print a memref_reinterpret_cast op of the form:
@@ -2859,69 +2917,110 @@ static ParseResult parseSubViewOp(OpAsmParser &parser, OperationState &result) {
   return parser.addTypeToList(dstType, result.types);
 }
 
-void mlir::SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
-                            ArrayRef<int64_t> staticOffsets,
-                            ArrayRef<int64_t> staticSizes,
-                            ArrayRef<int64_t> staticStrides, ValueRange offsets,
-                            ValueRange sizes, ValueRange strides,
+// Build a SubViewOp with mixed static and dynamic entries and custom result
+// type. If the type passed is nullptr, it is inferred.
+void mlir::SubViewOp::build(OpBuilder &b, OperationState &result,
+                            MemRefType resultType, Value source,
+                            ArrayRef<OpFoldResult> offsets,
+                            ArrayRef<OpFoldResult> sizes,
+                            ArrayRef<OpFoldResult> strides,
                             ArrayRef<NamedAttribute> attrs) {
+  SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
+  SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets,
+                             ShapedType::kDynamicStrideOrOffset);
+  dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes,
+                             ShapedType::kDynamicSize);
+  dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides,
+                             ShapedType::kDynamicStrideOrOffset);
   auto sourceMemRefType = source.getType().cast<MemRefType>();
-  auto resultType = inferResultType(sourceMemRefType, staticOffsets,
-                                    staticSizes, staticStrides);
-  build(b, result, resultType, source, offsets, sizes, strides,
-        b.getI64ArrayAttr(staticOffsets), b.getI64ArrayAttr(staticSizes),
-        b.getI64ArrayAttr(staticStrides));
+  // Structuring implementation this way avoids duplication between builders.
+  if (!resultType) {
+    resultType = SubViewOp::inferResultType(sourceMemRefType, staticOffsets,
+                                            staticSizes, staticStrides)
+                     .cast<MemRefType>();
+  }
+  build(b, result, resultType, source, dynamicOffsets, dynamicSizes,
+        dynamicStrides, b.getI64ArrayAttr(staticOffsets),
+        b.getI64ArrayAttr(staticSizes), b.getI64ArrayAttr(staticStrides));
   result.addAttributes(attrs);
 }
 
-/// Build a SubViewOp with all dynamic entries: `staticOffsets`, `staticSizes`
-/// and `staticStrides` are automatically filled with source-memref-rank
-/// sentinel values that encode dynamic entries.
+// Build a SubViewOp with mixed static and dynamic entries and inferred result
+// type.
 void mlir::SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
-                            ValueRange offsets, ValueRange sizes,
-                            ValueRange strides,
+                            ArrayRef<OpFoldResult> offsets,
+                            ArrayRef<OpFoldResult> sizes,
+                            ArrayRef<OpFoldResult> strides,
                             ArrayRef<NamedAttribute> attrs) {
-  auto sourceMemRefType = source.getType().cast<MemRefType>();
-  unsigned rank = sourceMemRefType.getRank();
-  SmallVector<int64_t, 4> staticOffsetsVector;
-  staticOffsetsVector.assign(rank, ShapedType::kDynamicStrideOrOffset);
-  SmallVector<int64_t, 4> staticSizesVector;
-  staticSizesVector.assign(rank, ShapedType::kDynamicSize);
-  SmallVector<int64_t, 4> staticStridesVector;
-  staticStridesVector.assign(rank, ShapedType::kDynamicStrideOrOffset);
-  build(b, result, source, staticOffsetsVector, staticSizesVector,
-        staticStridesVector, offsets, sizes, strides, attrs);
+  build(b, result, MemRefType(), source, offsets, sizes, strides, attrs);
 }
 
-/// Build a SubViewOp as above but with custom result type.
+// Build a SubViewOp with static entries and inferred result type.
+void mlir::SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
+                            ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes,
+                            ArrayRef<int64_t> strides,
+                            ArrayRef<NamedAttribute> attrs) {
+  SmallVector<OpFoldResult> offsetValues = llvm::to_vector<4>(
+      llvm::map_range(offsets, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  SmallVector<OpFoldResult> sizeValues =
+      llvm::to_vector<4>(llvm::map_range(sizes, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  build(b, result, source, offsetValues, sizeValues, strideValues, attrs);
+}
+
+// Build a SubViewOp with dynamic entries and custom result type. If the
+// type passed is nullptr, it is inferred.
 void mlir::SubViewOp::build(OpBuilder &b, OperationState &result,
                             MemRefType resultType, Value source,
-                            ArrayRef<int64_t> staticOffsets,
-                            ArrayRef<int64_t> staticSizes,
-                            ArrayRef<int64_t> staticStrides, ValueRange offsets,
-                            ValueRange sizes, ValueRange strides,
+                            ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes,
+                            ArrayRef<int64_t> strides,
                             ArrayRef<NamedAttribute> attrs) {
-  build(b, result, resultType, source, offsets, sizes, strides,
-        b.getI64ArrayAttr(staticOffsets), b.getI64ArrayAttr(staticSizes),
-        b.getI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
+  SmallVector<OpFoldResult> offsetValues = llvm::to_vector<4>(
+      llvm::map_range(offsets, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  SmallVector<OpFoldResult> sizeValues =
+      llvm::to_vector<4>(llvm::map_range(sizes, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [&](int64_t v) -> OpFoldResult {
+        return b.getI64IntegerAttr(v);
+      }));
+  build(b, result, resultType, source, offsetValues, sizeValues, strideValues,
+        attrs);
 }
 
-/// Build a SubViewOp as above but with custom result type.
+// Build a SubViewOp with dynamic entries and custom result type. If the type
+// passed is nullptr, it is inferred.
 void mlir::SubViewOp::build(OpBuilder &b, OperationState &result,
                             MemRefType resultType, Value source,
                             ValueRange offsets, ValueRange sizes,
                             ValueRange strides,
                             ArrayRef<NamedAttribute> attrs) {
-  auto sourceMemRefType = source.getType().cast<MemRefType>();
-  unsigned rank = sourceMemRefType.getRank();
-  SmallVector<int64_t, 4> staticOffsetsVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
-  SmallVector<int64_t, 4> staticStridesVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  build(b, result, resultType, source, staticOffsetsVector, staticSizesVector,
-        staticStridesVector, offsets, sizes, strides, attrs);
+  SmallVector<OpFoldResult> offsetValues = llvm::to_vector<4>(
+      llvm::map_range(offsets, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> sizeValues = llvm::to_vector<4>(
+      llvm::map_range(sizes, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [](Value v) -> OpFoldResult { return v; }));
+  build(b, result, resultType, source, offsetValues, sizeValues, strideValues);
+}
+
+// Build a SubViewOp with dynamic entries and inferred result type.
+void mlir::SubViewOp::build(OpBuilder &b, OperationState &result, Value source,
+                            ValueRange offsets, ValueRange sizes,
+                            ValueRange strides,
+                            ArrayRef<NamedAttribute> attrs) {
+  build(b, result, MemRefType(), source, offsets, sizes, strides, attrs);
 }
 
 /// For ViewLikeOpInterface.
@@ -3123,33 +3222,16 @@ SmallVector<Range, 8> mlir::getOrCreateRanges(OffsetSizeAndStrideOpInterface op,
 
 namespace {
 
-/// Take a list of `values` with potential new constant to extract and a list
-/// of `constantValues` with`values.size()` sentinel that evaluate to true by
-/// applying `isDynamic`.
 /// Detects the `values` produced by a ConstantIndexOp and places the new
 /// constant in place of the corresponding sentinel value.
-void canonicalizeSubViewPart(SmallVectorImpl<Value> &values,
-                             SmallVectorImpl<int64_t> &constantValues,
+void canonicalizeSubViewPart(SmallVectorImpl<OpFoldResult> &values,
                              llvm::function_ref<bool(int64_t)> isDynamic) {
-  bool hasNewStaticValue = llvm::any_of(
-      values, [](Value val) { return matchPattern(val, m_ConstantIndex()); });
-  if (hasNewStaticValue) {
-    for (unsigned cstIdx = 0, valIdx = 0, e = constantValues.size();
-         cstIdx != e; ++cstIdx) {
-      // Was already static, skip.
-      if (!isDynamic(constantValues[cstIdx]))
-        continue;
-      // Newly static, move from Value to constant.
-      if (matchPattern(values[valIdx], m_ConstantIndex())) {
-        constantValues[cstIdx] =
-            cast<ConstantIndexOp>(values[valIdx].getDefiningOp()).getValue();
-        // Erase for impl. simplicity. Reverse iterator if we really must.
-        values.erase(std::next(values.begin(), valIdx));
-        continue;
-      }
-      // Remains dynamic move to next value.
-      ++valIdx;
-    }
+  for (OpFoldResult &ofr : values) {
+    if (ofr.is<Attribute>())
+      continue;
+    // Newly static, move from Value to constant.
+    if (auto cstOp = ofr.dyn_cast<Value>().getDefiningOp<ConstantIndexOp>())
+      ofr = OpBuilder(cstOp).getIndexAttr(cstOp.getValue());
   }
 }
 
@@ -3180,32 +3262,16 @@ class OpWithOffsetSizesAndStridesConstantArgumentFolder final
 
     // At least one of offsets/sizes/strides is a new constant.
     // Form the new list of operands and constant attributes from the existing.
-    SmallVector<Value, 8> newOffsets(op.offsets());
-    SmallVector<int64_t, 8> newStaticOffsets =
-        extractFromI64ArrayAttr(op.static_offsets());
-    std::array<unsigned, 3> ranks = op.getArrayAttrRanks();
-    (void)ranks;
-    assert(newStaticOffsets.size() == ranks[0]);
-    canonicalizeSubViewPart(newOffsets, newStaticOffsets,
-                            ShapedType::isDynamicStrideOrOffset);
-
-    SmallVector<Value, 8> newSizes(op.sizes());
-    SmallVector<int64_t, 8> newStaticSizes =
-        extractFromI64ArrayAttr(op.static_sizes());
-    assert(newStaticSizes.size() == ranks[1]);
-    canonicalizeSubViewPart(newSizes, newStaticSizes, ShapedType::isDynamic);
-
-    SmallVector<Value, 8> newStrides(op.strides());
-    SmallVector<int64_t, 8> newStaticStrides =
-        extractFromI64ArrayAttr(op.static_strides());
-    assert(newStaticStrides.size() == ranks[2]);
-    canonicalizeSubViewPart(newStrides, newStaticStrides,
-                            ShapedType::isDynamicStrideOrOffset);
+    SmallVector<OpFoldResult> mixedOffsets(op.getMixedOffsets());
+    SmallVector<OpFoldResult> mixedSizes(op.getMixedSizes());
+    SmallVector<OpFoldResult> mixedStrides(op.getMixedStrides());
+    canonicalizeSubViewPart(mixedOffsets, ShapedType::isDynamicStrideOrOffset);
+    canonicalizeSubViewPart(mixedSizes, ShapedType::isDynamic);
+    canonicalizeSubViewPart(mixedStrides, ShapedType::isDynamicStrideOrOffset);
 
     // Create the new op in canonical form.
-    auto newOp = rewriter.create<OpType>(
-        op.getLoc(), op.source(), newStaticOffsets, newStaticSizes,
-        newStaticStrides, newOffsets, newSizes, newStrides);
+    auto newOp = rewriter.create<OpType>(op.getLoc(), op.source(), mixedOffsets,
+                                         mixedSizes, mixedStrides);
 
     replaceWithNewOp(rewriter, op, newOp);
 
@@ -3432,70 +3498,68 @@ Type SubTensorOp::inferResultType(RankedTensorType sourceRankedTensorType,
                                sourceRankedTensorType.getElementType());
 }
 
+// Build a SubTensorOp with mixed static and dynamic entries and custom result
+// type. If the type passed is nullptr, it is inferred.
 void mlir::SubTensorOp::build(OpBuilder &b, OperationState &result,
-                              Value source, ArrayRef<int64_t> staticOffsets,
-                              ArrayRef<int64_t> staticSizes,
-                              ArrayRef<int64_t> staticStrides,
-                              ValueRange offsets, ValueRange sizes,
-                              ValueRange strides,
+                              RankedTensorType resultType, Value source,
+                              ArrayRef<OpFoldResult> offsets,
+                              ArrayRef<OpFoldResult> sizes,
+                              ArrayRef<OpFoldResult> strides,
                               ArrayRef<NamedAttribute> attrs) {
+  SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
+  SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets,
+                             ShapedType::kDynamicStrideOrOffset);
+  dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes,
+                             ShapedType::kDynamicSize);
+  dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides,
+                             ShapedType::kDynamicStrideOrOffset);
   auto sourceRankedTensorType = source.getType().cast<RankedTensorType>();
-  auto resultType = inferResultType(sourceRankedTensorType, staticOffsets,
-                                    staticSizes, staticStrides);
-  build(b, result, resultType, source, offsets, sizes, strides,
-        b.getI64ArrayAttr(staticOffsets), b.getI64ArrayAttr(staticSizes),
-        b.getI64ArrayAttr(staticStrides));
+  // Structuring implementation this way avoids duplication between builders.
+  if (!resultType) {
+    resultType =
+        SubTensorOp::inferResultType(sourceRankedTensorType, staticOffsets,
+                                     staticSizes, staticStrides)
+            .cast<RankedTensorType>();
+  }
+  build(b, result, resultType, source, dynamicOffsets, dynamicSizes,
+        dynamicStrides, b.getI64ArrayAttr(staticOffsets),
+        b.getI64ArrayAttr(staticSizes), b.getI64ArrayAttr(staticStrides));
   result.addAttributes(attrs);
 }
 
-/// Build a SubTensorOp with all dynamic entries: `staticOffsets`, `staticSizes`
-/// and `staticStrides` are automatically filled with sentinel values that
-/// encode dynamic entries.
+// Build a SubTensorOp with mixed static and dynamic entries and inferred result
+// type.
 void mlir::SubTensorOp::build(OpBuilder &b, OperationState &result,
-                              Value source, ValueRange offsets,
-                              ValueRange sizes, ValueRange strides,
+                              Value source, ArrayRef<OpFoldResult> offsets,
+                              ArrayRef<OpFoldResult> sizes,
+                              ArrayRef<OpFoldResult> strides,
                               ArrayRef<NamedAttribute> attrs) {
-  auto sourceRankedTensorType = source.getType().cast<RankedTensorType>();
-  unsigned rank = sourceRankedTensorType.getRank();
-  SmallVector<int64_t, 4> staticOffsetsVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
-  SmallVector<int64_t, 4> staticStridesVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  build(b, result, source, staticOffsetsVector, staticSizesVector,
-        staticStridesVector, offsets, sizes, strides, attrs);
+  build(b, result, RankedTensorType(), source, offsets, sizes, strides, attrs);
 }
 
-/// Build a SubTensorOp as above but with custom result type.
+// Build a SubTensorOp with dynamic entries and custom result type. If the type
+// passed is nullptr, it is inferred.
 void mlir::SubTensorOp::build(OpBuilder &b, OperationState &result,
                               RankedTensorType resultType, Value source,
-                              ArrayRef<int64_t> staticOffsets,
-                              ArrayRef<int64_t> staticSizes,
-                              ArrayRef<int64_t> staticStrides,
                               ValueRange offsets, ValueRange sizes,
                               ValueRange strides,
                               ArrayRef<NamedAttribute> attrs) {
-  build(b, result, resultType, source, offsets, sizes, strides,
-        b.getI64ArrayAttr(staticOffsets), b.getI64ArrayAttr(staticSizes),
-        b.getI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
+  SmallVector<OpFoldResult> offsetValues = llvm::to_vector<4>(
+      llvm::map_range(offsets, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> sizeValues = llvm::to_vector<4>(
+      llvm::map_range(sizes, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [](Value v) -> OpFoldResult { return v; }));
+  build(b, result, resultType, source, offsetValues, sizeValues, strideValues);
 }
 
-/// Build a SubTensorOp as above but with custom result type.
+// Build a SubTensorOp with dynamic entries and inferred result type.
 void mlir::SubTensorOp::build(OpBuilder &b, OperationState &result,
-                              RankedTensorType resultType, Value source,
-                              ValueRange offsets, ValueRange sizes,
-                              ValueRange strides,
+                              Value source, ValueRange offsets,
+                              ValueRange sizes, ValueRange strides,
                               ArrayRef<NamedAttribute> attrs) {
-  auto sourceRankedTensorType = source.getType().cast<RankedTensorType>();
-  unsigned rank = sourceRankedTensorType.getRank();
-  SmallVector<int64_t, 4> staticOffsetsVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
-  SmallVector<int64_t, 4> staticStridesVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  build(b, result, resultType, source, staticOffsetsVector, staticSizesVector,
-        staticStridesVector, offsets, sizes, strides, attrs);
+  build(b, result, RankedTensorType(), source, offsets, sizes, strides, attrs);
 }
 
 /// Verifier for SubTensorOp.
@@ -3563,34 +3627,40 @@ static ParseResult parseSubTensorInsertOp(OpAsmParser &parser,
   return parser.addTypeToList(dstType, result.types);
 }
 
-void mlir::SubTensorInsertOp::build(
-    OpBuilder &b, OperationState &result, Value source, Value dest,
-    ArrayRef<int64_t> staticOffsets, ArrayRef<int64_t> staticSizes,
-    ArrayRef<int64_t> staticStrides, ValueRange offsets, ValueRange sizes,
-    ValueRange strides, ArrayRef<NamedAttribute> attrs) {
-  build(b, result, dest.getType(), source, dest, offsets, sizes, strides,
-        b.getI64ArrayAttr(staticOffsets), b.getI64ArrayAttr(staticSizes),
-        b.getI64ArrayAttr(staticStrides));
+// Build a SubTensorInsertOp with mixed static and dynamic entries.
+void mlir::SubTensorInsertOp::build(OpBuilder &b, OperationState &result,
+                                    Value source, Value dest,
+                                    ArrayRef<OpFoldResult> offsets,
+                                    ArrayRef<OpFoldResult> sizes,
+                                    ArrayRef<OpFoldResult> strides,
+                                    ArrayRef<NamedAttribute> attrs) {
+  SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
+  SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets,
+                             ShapedType::kDynamicStrideOrOffset);
+  dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes,
+                             ShapedType::kDynamicSize);
+  dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides,
+                             ShapedType::kDynamicStrideOrOffset);
+  build(b, result, dest.getType(), source, dest, dynamicOffsets, dynamicSizes,
+        dynamicStrides, b.getI64ArrayAttr(staticOffsets),
+        b.getI64ArrayAttr(staticSizes), b.getI64ArrayAttr(staticStrides));
   result.addAttributes(attrs);
 }
 
-/// Build a SubViewOp with all dynamic entries: `staticOffsets`, `staticSizes`
-/// and `staticStrides` are automatically filled with source-memref-rank
-/// sentinel values that encode dynamic entries.
+// Build a SubTensorInsertOp with dynamic entries.
 void mlir::SubTensorInsertOp::build(OpBuilder &b, OperationState &result,
                                     Value source, Value dest,
                                     ValueRange offsets, ValueRange sizes,
                                     ValueRange strides,
                                     ArrayRef<NamedAttribute> attrs) {
-  auto destRankedTensorType = dest.getType().cast<RankedTensorType>();
-  unsigned rank = destRankedTensorType.getRank();
-  SmallVector<int64_t, 4> staticOffsetsVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  SmallVector<int64_t, 4> staticSizesVector(rank, ShapedType::kDynamicSize);
-  SmallVector<int64_t, 4> staticStridesVector(
-      rank, ShapedType::kDynamicStrideOrOffset);
-  build(b, result, source, dest, staticOffsetsVector, staticSizesVector,
-        staticStridesVector, offsets, sizes, strides, attrs);
+  SmallVector<OpFoldResult> offsetValues = llvm::to_vector<4>(
+      llvm::map_range(offsets, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> sizeValues = llvm::to_vector<4>(
+      llvm::map_range(sizes, [](Value v) -> OpFoldResult { return v; }));
+  SmallVector<OpFoldResult> strideValues = llvm::to_vector<4>(
+      llvm::map_range(strides, [](Value v) -> OpFoldResult { return v; }));
+  build(b, result, source, dest, offsetValues, sizeValues, strideValues);
 }
 
 /// Verifier for SubViewOp.
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp
index 27e61dc7a3ac..9d1378f06bcb 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/ExpandOps.cpp
@@ -83,7 +83,7 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<MemRefReshapeOp> {
       return failure();
 
     int64_t rank = shapeType.cast<MemRefType>().getDimSize(0);
-    SmallVector<Value, 4> sizes, strides;
+    SmallVector<OpFoldResult, 4> sizes, strides;
     sizes.resize(rank);
     strides.resize(rank);
 
@@ -99,12 +99,9 @@ struct MemRefReshapeOpConverter : public OpRewritePattern<MemRefReshapeOp> {
       if (i > 0)
         stride = rewriter.create<MulIOp>(loc, stride, size);
     }
-    SmallVector<int64_t, 2> staticSizes(rank, ShapedType::kDynamicSize);
-    SmallVector<int64_t, 2> staticStrides(rank,
-                                          ShapedType::kDynamicStrideOrOffset);
     rewriter.replaceOpWithNewOp<MemRefReinterpretCastOp>(
-        op, op.getType(), op.source(), /*staticOffset = */ 0, staticSizes,
-        staticStrides, /*offset=*/llvm::None, sizes, strides);
+        op, op.getType(), op.source(), /*offset=*/rewriter.getIndexAttr(0),
+        sizes, strides);
     return success();
   }
 };
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index ca6e92d95ed0..a567ba8f37a0 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -2233,10 +2233,9 @@ static Value createScopedSubViewIntersection(VectorTransferOpInterface xferOp,
   // TODO: relax this precondition, will require rank-reducing subviews.
   assert(memrefRank == alloc.getType().cast<MemRefType>().getRank() &&
          "Expected memref rank to match the alloc rank");
-  Value one = std_constant_index(1);
   ValueRange leadingIndices =
       xferOp.indices().take_front(xferOp.getLeadingShapedRank());
-  SmallVector<Value, 4> sizes;
+  SmallVector<OpFoldResult, 4> sizes;
   sizes.append(leadingIndices.begin(), leadingIndices.end());
   xferOp.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
     using MapList = ArrayRef<ArrayRef<AffineExpr>>;
@@ -2252,8 +2251,12 @@ static Value createScopedSubViewIntersection(VectorTransferOpInterface xferOp,
                                  ValueRange{dimMemRef, index, dimAlloc});
     sizes.push_back(affineMin);
   });
-  return std_sub_view(xferOp.source(), xferOp.indices(), sizes,
-                      SmallVector<Value, 4>(memrefRank, one));
+
+  SmallVector<OpFoldResult, 4> indices = llvm::to_vector<4>(llvm::map_range(
+      xferOp.indices(), [](Value idx) -> OpFoldResult { return idx; }));
+  return std_sub_view(
+      xferOp.source(), indices, sizes,
+      SmallVector<OpFoldResult>(memrefRank, OpBuilder(xferOp).getIndexAttr(1)));
 }
 
 /// Given an `xferOp` for which:
diff --git a/mlir/test/Dialect/Linalg/fusion-sequence.mlir b/mlir/test/Dialect/Linalg/fusion-sequence.mlir
index 2738eb0f9114..97d81378701e 100644
--- a/mlir/test/Dialect/Linalg/fusion-sequence.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-sequence.mlir
@@ -238,7 +238,7 @@ module {
 //  CHECK-SAME:         ins(%[[T1]], %[[ARG5]]
 //  CHECK-SAME:         ) outs(%[[STARG6]] : tensor<?x?xf32>)
 //       CHECK:       %[[R1:.+]] = subtensor_insert %[[T2]]
-//  CHECK-SAME:         into %[[ARG8]][%[[IV0]], %[[C0]]]
+//  CHECK-SAME:         into %[[ARG8]][%[[IV0]], 0]
 //       CHECK:       scf.yield %[[R1]]
 //       CHECK:     }
 //       CHECK:     return %[[R0]]
diff --git a/mlir/test/Dialect/Linalg/hoist-padding.mlir b/mlir/test/Dialect/Linalg/hoist-padding.mlir
new file mode 100644
index 000000000000..27750ea8a024
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/hoist-padding.mlir
@@ -0,0 +1,85 @@
+// RUN: mlir-opt %s -test-linalg-transform-patterns=test-hoist-padding-2-level -canonicalize | FileCheck %s
+
+#map0 = affine_map<(d0)[s0] -> (2, -d0 + s0)>
+#map1 = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+#map2 = affine_map<(d0)[s0] -> (3, -d0 + s0)>
+#map3 = affine_map<(d0, d1) -> (2, d0 - d1)>
+#map4 = affine_map<(d0, d1) -> (3, d0 - d1)>
+
+// CHECK-LABEL: func @matmul_tensors
+func @matmul_tensors(
+  %arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<?x?xf32>)
+  -> tensor<?x?xf32>
+{
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %c4 = constant 4 : index
+  %cst = constant 0.000000e+00 : f32
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %0 = dim %arg0, %c0 : tensor<?x?xf32>
+  %1 = dim %arg0, %c1 : tensor<?x?xf32>
+  %2 = dim %arg1, %c1 : tensor<?x?xf32>
+
+  //      CHECK: scf.for
+  //      CHECK:   linalg.init_tensor [%{{.*}}, 2, 4] : tensor<?x2x4xf32>
+  // 1-D loop
+  //      CHECK:   %[[A:.*]] = scf.for
+  //  CHECK-NOT:     scf.for
+  //      CHECK:     subtensor %{{.*}} [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+  //      CHECK:     linalg.simple_pad %{{.*}} : tensor<?x?xf32> to tensor<2x4xf32> pad f32
+  //      CHECK:     subtensor_insert %{{.*}} into %{{.*}}[%{{.*}}, 0, 0]
+  // CHECK-SAME:       [1, 2, 4] [1, 1, 1] : tensor<2x4xf32> into tensor<?x2x4xf32>
+  // 2-D loop
+  //      CHECK:   linalg.init_tensor [%{{.*}}, %{{.*}}, 4, 3] : tensor<?x?x4x3xf32>
+  //      CHECK:   %[[B:.*]] = scf.for
+  //      CHECK:     scf.for
+  //  CHECK-NOT:       scf.for
+  //      CHECK:       subtensor %{{.*}} [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+  //      CHECK:       linalg.simple_pad %{{.*}} : tensor<?x?xf32> to tensor<4x3xf32> pad f32
+  //      CHECK:       subtensor_insert %{{.*}} into %{{.*}}[%{{.*}}, %{{.*}}, 0, 0]
+  // CHECK-SAME:         [1, 1, 4, 3] [1, 1, 1, 1] : tensor<4x3xf32> into tensor<?x?x4x3xf32>
+  // 2-D loop
+  //      CHECK:   scf.for %[[J:[0-9a-zA-Z]+]]
+  //      CHECK:     scf.for %[[K:[0-9a-zA-Z]+]]
+  //  CHECK-NOT:       scf.for
+  //      CHECK:       %[[stA:.*]] = subtensor %[[A]][%[[K]], 0, 0] [1, 2, 4] [1, 1, 1] :
+  // CHECK-SAME:         tensor<?x2x4xf32> to tensor<2x4xf32>
+  //      CHECK:       %[[stB:.*]] = subtensor %[[B]][%[[K]], %[[J]], 0, 0] [1, 1, 4, 3] [1, 1, 1, 1] :
+  // CHECK-SAME:         tensor<?x?x4x3xf32> to tensor<4x3xf32>
+  //      CHECK:       %[[stC:.*]] = linalg.simple_pad %{{.*}} pad %{{.*}} :
+  // CHECK-SAME:         tensor<?x?xf32> to tensor<2x3xf32> pad f32
+  //      CHECK:       linalg.matmul ins(%[[stA]], %[[stB]] : tensor<2x4xf32>, tensor<4x3xf32>)
+  // CHECK-SAME:         outs(%[[stC]] : tensor<2x3xf32>) -> tensor<2x3xf32>
+  %3 = scf.for %arg3 = %c0 to %0 step %c2 iter_args(%arg4 = %arg2) -> (tensor<?x?xf32>) {
+    %4 = scf.for %arg5 = %c0 to %2 step %c3 iter_args(%arg6 = %arg4) -> (tensor<?x?xf32>) {
+      %5 = scf.for %arg7 = %c0 to %1 step %c4 iter_args(%arg8 = %arg6) -> (tensor<?x?xf32>) {
+        %6 = dim %arg0, %c0 : tensor<?x?xf32>
+        %7 = affine.min #map0(%arg3)[%6]
+        %8 = dim %arg0, %c1 : tensor<?x?xf32>
+        %9 = affine.min #map1(%arg7)[%8]
+        %10 = subtensor %arg0[%arg3, %arg7] [%7, %9] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+        %11 = dim %arg1, %c0 : tensor<?x?xf32>
+        %12 = affine.min #map1(%arg7)[%11]
+        %13 = dim %arg1, %c1 : tensor<?x?xf32>
+        %14 = affine.min #map2(%arg5)[%13]
+        %15 = subtensor %arg1[%arg7, %arg5] [%12, %14] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+        %16 = dim %arg8, %c0 : tensor<?x?xf32>
+        %17 = affine.min #map3(%16, %arg3)
+        %18 = dim %arg8, %c1 : tensor<?x?xf32>
+        %19 = affine.min #map4(%18, %arg5)
+        %20 = subtensor %arg8[%arg3, %arg5] [%17, %19] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+        %21 = linalg.simple_pad %10 pad %cst : tensor<?x?xf32> to tensor<2x4xf32> pad f32
+        %22 = linalg.simple_pad %15 pad %cst : tensor<?x?xf32> to tensor<4x3xf32> pad f32
+        %23 = linalg.simple_pad %20 pad %cst : tensor<?x?xf32> to tensor<2x3xf32> pad f32
+        %24 = linalg.matmul ins(%21, %22 : tensor<2x4xf32>, tensor<4x3xf32>) outs(%23 : tensor<2x3xf32>) -> tensor<2x3xf32>
+        %25 = subtensor %24[0, 0] [%7, %14] [1, 1] : tensor<2x3xf32> to tensor<?x?xf32>
+        %26 = subtensor_insert %25 into %arg8[%arg3, %arg5] [%17, %19] [%c1, %c1] : tensor<?x?xf32> into tensor<?x?xf32>
+        scf.yield %26 : tensor<?x?xf32>
+      }
+      scf.yield %5 : tensor<?x?xf32>
+    }
+    scf.yield %4 : tensor<?x?xf32>
+  }
+  return %3 : tensor<?x?xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/promote.mlir b/mlir/test/Dialect/Linalg/promote.mlir
index 7988abbeaf43..838699301639 100644
--- a/mlir/test/Dialect/Linalg/promote.mlir
+++ b/mlir/test/Dialect/Linalg/promote.mlir
@@ -7,7 +7,6 @@
 #map3 = affine_map<(d0) -> (d0 + 3)>
 
 // CHECK-DAG: #[[$strided2D:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
-// CHECK-DAG: #[[$strided2D_dynamic:.*]] = affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + s0 + d1 * s2)>
 
 func @matmul_f32(%A: memref<?xi8>, %M: index, %N: index, %K: index) {
   %c4 = constant 4 : index
@@ -49,28 +48,28 @@ func @matmul_f32(%A: memref<?xi8>, %M: index, %N: index, %K: index) {
 //      ALLOCA:         %[[tmpA:.*]] = alloca() : memref<32xi8>
 //       CHECK:         %[[fullA:.*]] = std.view %[[tmpA]][{{.*}}][{{.*}}] : memref<32xi8> to memref<?x?xf32>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf32>
-//       CHECK:         %[[partialA:.*]] = subview %[[fullA]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D_dynamic]]>
+//       CHECK:         %[[partialA:.*]] = subview %[[fullA]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D]]>
 ///
 //       CHECK:         %[[tmpB:.*]] = alloc() : memref<48xi8>
 //      ALLOCA:         %[[tmpB:.*]] = alloca() : memref<48xi8>
 //       CHECK:         %[[fullB:.*]] = std.view %[[tmpB]][{{.*}}][{{.*}}] : memref<48xi8> to memref<?x?xf32>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf32>
-//       CHECK:         %[[partialB:.*]] = subview %[[fullB]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D_dynamic]]>
+//       CHECK:         %[[partialB:.*]] = subview %[[fullB]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D]]>
 ///
 //       CHECK:         %[[tmpC:.*]] = alloc() : memref<24xi8>
 //      ALLOCA:         %[[tmpC:.*]] = alloca() : memref<24xi8>
 //       CHECK:         %[[fullC:.*]] = std.view %[[tmpC]][{{.*}}][{{.*}}] : memref<24xi8> to memref<?x?xf32>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf32>
-//       CHECK:         %[[partialC:.*]] = subview %[[fullC]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D_dynamic]]>
+//       CHECK:         %[[partialC:.*]] = subview %[[fullC]]{{.*}} : memref<?x?xf32> to memref<?x?xf32, #[[$strided2D]]>
 
-//       CHECK:         linalg.copy(%[[vA]], %[[partialA]]) : memref<?x?xf32, #[[$strided2D]]>, memref<?x?xf32, #[[$strided2D_dynamic]]>
-//       CHECK:         linalg.copy(%[[vB]], %[[partialB]]) : memref<?x?xf32, #[[$strided2D]]>, memref<?x?xf32, #[[$strided2D_dynamic]]>
-//       CHECK:         linalg.copy(%[[vC]], %[[partialC]]) : memref<?x?xf32, #[[$strided2D]]>, memref<?x?xf32, #[[$strided2D_dynamic]]>
+//       CHECK:         linalg.copy(%[[vA]], %[[partialA]]) : memref<?x?xf32, #[[$strided2D]]>, memref<?x?xf32, #[[$strided2D]]>
+//       CHECK:         linalg.copy(%[[vB]], %[[partialB]]) : memref<?x?xf32, #[[$strided2D]]>, memref<?x?xf32, #[[$strided2D]]>
+//       CHECK:         linalg.copy(%[[vC]], %[[partialC]]) : memref<?x?xf32, #[[$strided2D]]>, memref<?x?xf32, #[[$strided2D]]>
 //
 //       CHECK:         linalg.matmul ins(%[[partialA]], %[[partialB]]{{.*}} outs(%[[partialC]]
 //
 //       CHECK:         linalg.copy(%[[partialC]], %[[vC]]) :
-//       CHECK:           memref<?x?xf32, #[[$strided2D_dynamic]]>,
+//       CHECK:           memref<?x?xf32, #[[$strided2D]]>,
 //       CHECK:           memref<?x?xf32, #[[$strided2D]]>
 //
 //       CHECK:         dealloc %[[tmpA]] : memref<32xi8>
@@ -121,26 +120,26 @@ func @matmul_f64(%A: memref<?xi8>, %M: index, %N: index, %K: index) {
 //       CHECK:         %[[tmpA_f64:.*]] = alloc() : memref<64xi8>
 //       CHECK:         %[[fullA_f64:.*]] = std.view %[[tmpA_f64]][{{.*}}][{{.*}}] : memref<64xi8> to memref<?x?xf64>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf64>
-//       CHECK:         %[[partialA_f64:.*]] = subview %[[fullA_f64]][%{{.*}}, %{{.*}}] : memref<?x?xf64> to memref<?x?xf64, #[[$strided2D_dynamic]]>
+//       CHECK:         %[[partialA_f64:.*]] = subview %[[fullA_f64]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf64> to memref<?x?xf64, #[[$strided2D]]>
 ///
 //       CHECK:         %[[tmpB_f64:.*]] = alloc() : memref<96xi8>
 //       CHECK:         %[[fullB_f64:.*]] = std.view %[[tmpB_f64]][{{.*}}][{{.*}}] : memref<96xi8> to memref<?x?xf64>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf64>
-//       CHECK:         %[[partialB_f64:.*]] = subview %[[fullB_f64]][%{{.*}}, %{{.*}}] : memref<?x?xf64> to memref<?x?xf64, #[[$strided2D_dynamic]]>
+//       CHECK:         %[[partialB_f64:.*]] = subview %[[fullB_f64]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf64> to memref<?x?xf64, #[[$strided2D]]>
 ///
 //       CHECK:         %[[tmpC_f64:.*]] = alloc() : memref<48xi8>
 //       CHECK:         %[[fullC_f64:.*]] = std.view %[[tmpC_f64]][{{.*}}][{{.*}}] : memref<48xi8> to memref<?x?xf64>
 //     DYNAMIC:         std.view %{{.*}}[{{.*}}][{{.*}}] : memref<?xi8> to memref<?x?xf64>
-//       CHECK:         %[[partialC_f64:.*]] = subview %[[fullC_f64]][%{{.*}}, %{{.*}}] : memref<?x?xf64> to memref<?x?xf64, #[[$strided2D_dynamic]]>
+//       CHECK:         %[[partialC_f64:.*]] = subview %[[fullC_f64]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf64> to memref<?x?xf64, #[[$strided2D]]>
 
-//       CHECK:         linalg.copy(%[[vA_f64]], %[[partialA_f64]]) : memref<?x?xf64, #[[$strided2D]]>, memref<?x?xf64, #[[$strided2D_dynamic]]>
-//       CHECK:         linalg.copy(%[[vB_f64]], %[[partialB_f64]]) : memref<?x?xf64, #[[$strided2D]]>, memref<?x?xf64, #[[$strided2D_dynamic]]>
-//       CHECK:         linalg.copy(%[[vC_f64]], %[[partialC_f64]]) : memref<?x?xf64, #[[$strided2D]]>, memref<?x?xf64, #[[$strided2D_dynamic]]>
+//       CHECK:         linalg.copy(%[[vA_f64]], %[[partialA_f64]]) : memref<?x?xf64, #[[$strided2D]]>, memref<?x?xf64, #[[$strided2D]]>
+//       CHECK:         linalg.copy(%[[vB_f64]], %[[partialB_f64]]) : memref<?x?xf64, #[[$strided2D]]>, memref<?x?xf64, #[[$strided2D]]>
+//       CHECK:         linalg.copy(%[[vC_f64]], %[[partialC_f64]]) : memref<?x?xf64, #[[$strided2D]]>, memref<?x?xf64, #[[$strided2D]]>
 //
 //       CHECK:         linalg.matmul ins(%[[partialA_f64]], %[[partialB_f64]]{{.*}} outs(%[[partialC_f64]]
 //
 //       CHECK:         linalg.copy(%[[partialC_f64]], %[[vC_f64]]) :
-//       CHECK:           memref<?x?xf64, #[[$strided2D_dynamic]]>,
+//       CHECK:           memref<?x?xf64, #[[$strided2D]]>,
 //       CHECK:           memref<?x?xf64, #[[$strided2D]]>
 //
 //       CHECK:         dealloc %[[tmpA_f64]] : memref<64xi8>
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index 44743eaedc8c..6dc0768bc2e3 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -814,3 +814,13 @@ func @fill_tensor(%arg0 : index, %arg1 : index, %arg2 : f32) -> tensor<?x?xf32>
   return %1 : tensor<?x?xf32>
 }
 // CHECK: %{{.+}} = linalg.fill(%{{.+}}, %{{.+}}) : tensor<?x?xf32>, f32 -> tensor<?x?xf32>
+
+// -----
+
+// TODO: this op should disappear once pad_tensors is available and connected.
+// CHECK-LABEL: func @simple_pad
+func @simple_pad(%0: tensor<?x4x?xf32>, %pad: f32) {
+//     CHECK:   linalg.simple_pad %{{.+}} pad %{{.+}}: tensor<?x4x?xf32> to tensor<8x4x8xf32>
+  %1 = linalg.simple_pad %0 pad %pad: tensor<?x4x?xf32> to tensor<8x4x8xf32> pad f32
+  return
+}
diff --git a/mlir/test/Dialect/Linalg/tile-and-pad-tensors.mlir b/mlir/test/Dialect/Linalg/tile-and-pad-tensors.mlir
new file mode 100644
index 000000000000..1291b5c990df
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/tile-and-pad-tensors.mlir
@@ -0,0 +1,39 @@
+// RUN: mlir-opt %s -test-linalg-transform-patterns=test-tile-and-pad-pattern -canonicalize
+//| FileCheck %s
+
+// CHECK-LABEL: func @matmul_tensors(
+// CHECK-SAME:    %[[TA:[0-9a-z]+]]: tensor<?x?xf32>
+// CHECK-SAME:    %[[TB:[0-9a-z]+]]: tensor<?x?xf32>
+// CHECK-SAME:    %[[TC:[0-9a-z]+]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
+func @matmul_tensors(
+  %arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<?x?xf32>)
+    -> tensor<?x?xf32> {
+//      CHECK: %[[TD0:.*]] = scf.for {{.*}} to {{.*}} step {{.*}} iter_args(%[[TC0:.*]] = %[[TC]]) -> (tensor<?x?xf32>) {
+//      CHECK:   %[[TD1:.*]] = scf.for {{.*}} to {{.*}} step {{.*}} iter_args(%[[TC1:.*]] = %[[TC0]]) -> (tensor<?x?xf32>) {
+//      CHECK:     %[[TD2:.*]] = scf.for {{.*}} to {{.*}} step {{.*}} iter_args(%[[TC2:.*]] = %[[TC1]]) -> (tensor<?x?xf32>) {
+//      CHECK:       %[[sTA:.*]] = subtensor %[[TA]][{{.*}}] : tensor<?x?xf32> to tensor<?x?xf32>
+//      CHECK:       %[[sTB:.*]] = subtensor %[[TB]][{{.*}}] : tensor<?x?xf32> to tensor<?x?xf32>
+//      CHECK:       %[[sTC:.*]] = subtensor %[[TC2]][{{.*}}] : tensor<?x?xf32> to tensor<?x?xf32>
+
+// Dynamic op has been canonicalized away.
+//  CHECK-NOT:       linalg.matmul {{.*}} tensor<?x?xf32>
+
+// Padding injects static information.
+//      CHECK:       %[[pA:.*]] = linalg.simple_pad %[[sTA]] pad %{{.*}} : tensor<?x?xf32> to tensor<2x4xf32> pad f32
+//      CHECK:       %[[pB:.*]] = linalg.simple_pad %[[sTB]] pad %{{.*}} : tensor<?x?xf32> to tensor<4x3xf32> pad f32
+//      CHECK:       %[[pC:.*]] = linalg.simple_pad %[[sTC]] pad %{{.*}} : tensor<?x?xf32> to tensor<2x3xf32> pad f32
+//      CHECK:       %[[pD:.*]] = linalg.matmul ins(%[[pA]], %[[pB]] : tensor<2x4xf32>, tensor<4x3xf32>)
+// CHECK-SAME:                                  outs(%[[pC]] : tensor<2x3xf32>)  -> tensor<2x3xf32>
+//      CHECK:       %[[sTD:.*]] = subtensor %[[pD]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : tensor<2x3xf32> to tensor<?x?xf32>
+//      CHECK:       %[[TD:.*]] = subtensor_insert %[[sTD]] into %[[TC2]][{{.*}}]  : tensor<?x?xf32> into tensor<?x?xf32>
+//      CHECK:       scf.yield %[[TD]] : tensor<?x?xf32>
+//      CHECK:     scf.yield %[[TD2]] : tensor<?x?xf32>
+//      CHECK:   scf.yield %[[TD1]] : tensor<?x?xf32>
+  %0 = linalg.matmul {__internal_linalg_transform__ = "tile-and-pad"}
+      ins(%arg0, %arg1: tensor<?x?xf32>, tensor<?x?xf32>)
+     outs(%arg2: tensor<?x?xf32>)
+    -> tensor<?x?xf32>
+
+//      CHECK: return %[[TD0]] : tensor<?x?xf32>
+  return %0 : tensor<?x?xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/transform-patterns.mlir b/mlir/test/Dialect/Linalg/transform-patterns.mlir
index 83cb16ba0e3e..32d2e01a9731 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s -test-linalg-transform-patterns=test-patterns | FileCheck %s
 
-// CHECK-DAG: #[[$STRIDED_1D:.*]] = affine_map<(d0)[s0, s1] -> (d0 * s1  + s0)>
+// CHECK-DAG: #[[$STRIDED_1D:.*]] = affine_map<(d0)[s0] -> (d0 + s0)>
 // Map corresponding to a 2D memory access where the stride along the last dim is known to be 1.
 // CHECK-DAG: #[[$STRIDED_2D_u_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 // Map corresponding to a 2D memory access where the stride along all dims are unknown.
@@ -48,7 +48,7 @@ func @matvec(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 // CHECK:         scf.parallel {{.*}} step (%[[c5]])
 // CHECK:           scf.for {{.*}} step %[[c6]]
 // CHECK:             linalg.matvec
-// CHECK:               ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>, memref<?xf32, #[[$STRIDED_1D]]>)
+// CHECK:               ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D_u_1]]>, memref<?xf32, #[[$STRIDED_1D]]>)
 // CHECK:              outs({{.*}}: memref<?xf32, #[[$STRIDED_1D]]>)
 
 func @matmul(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
@@ -87,8 +87,8 @@ func @matmul(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 // CHECK:                             scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c3]] {
 // CHECK:                               scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c4]] {
 // CHECK:                                 linalg.matmul
-// CHECK:                                   ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>, memref<?x?xf32, #[[$STRIDED_2D]]>)
-// CHECK:                                  outs({{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>)
+// CHECK:                                   ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D_u_1]]>, memref<?x?xf32, #[[$STRIDED_2D_u_1]]>)
+// CHECK:                                  outs({{.*}}: memref<?x?xf32, #[[$STRIDED_2D_u_1]]>)
 
 #matmul_accesses = [
   affine_map<(m, n, k) -> (m, k)>,
@@ -172,7 +172,7 @@ func @matvec_perm(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 // CHECK:         scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c6]]
 // CHECK:           scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c5]]
 // CHECK:             linalg.matvec
-// CHECK:               ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>, memref<?xf32, #[[$STRIDED_1D]]>)
+// CHECK:               ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D_u_1]]>, memref<?xf32, #[[$STRIDED_1D]]>)
 // CHECK:              outs({{.*}}: memref<?xf32, #[[$STRIDED_1D]]>)
 
 func @matmul_perm(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
@@ -205,8 +205,8 @@ func @matmul_perm(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 // CHECK:                       scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c30]] {
 // CHECK:                         scf.for {{.*}} = %[[c0]] to {{.*}} step %[[c40]] {
 // CHECK:                                 linalg.matmul
-// CHECK:                                  ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>, memref<?x?xf32, #[[$STRIDED_2D]]>)
-// CHECK:                                   outs({{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>)
+// CHECK:                                  ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D_u_1]]>, memref<?x?xf32, #[[$STRIDED_2D_u_1]]>)
+// CHECK:                                   outs({{.*}}: memref<?x?xf32, #[[$STRIDED_2D_u_1]]>)
 
 func @promote_subview_matmul(%arg0: memref<?x?xf32, offset: ?, strides: [?, 1]>,
                              %arg1: memref<?x?xf32, offset: ?, strides: [?, 1]>,
@@ -250,13 +250,16 @@ func @promote_subview_matmul(%arg0: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 // CHECK:               %[[s2:.*]] = subview {{%.*}}[{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32, #map{{.*}}> to memref<?x?xf32, #map{{.*}}>
 // CHECK:               %[[a0:.*]] = alloc({{%.*}}) : memref<?xi8>
 // CHECK:               %[[v0:.*]] = std.view %[[a0]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK:               %[[l0:.*]] = subview %[[v0]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK:               %[[l0:.*]] = subview %[[v0]][0, 0] [%{{.*}}, %{{.*}}] [1, 1]
+// CHECK-SAME:            memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK:               %[[a1:.*]] = alloc({{%.*}}) : memref<?xi8>
 // CHECK:               %[[v1:.*]] = std.view %[[a1]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK:               %[[l1:.*]] = subview %[[v1]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK:               %[[l1:.*]] = subview %[[v1]][0, 0] [%{{.*}}, %{{.*}}] [1, 1]
+// CHECK-SAME:            memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK:               %[[a2:.*]] = alloc({{%.*}}) : memref<?xi8>
 // CHECK:               %[[v2:.*]] = std.view %[[a2]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK:               %[[l2:.*]] = subview %[[v2]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK:               %[[l2:.*]] = subview %[[v2]][0, 0] [%{{.*}}, %{{.*}}] [1, 1]
+// CHECK-SAME:            memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK:               linalg.copy(%[[s0]], %[[l0]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>
 // CHECK:               linalg.copy(%[[s1]], %[[l1]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>
 // CHECK:               linalg.copy(%[[s2]], %[[l2]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>
@@ -306,13 +309,13 @@ func @promote_first_subview_matmul(%arg0: memref<?x?xf32, offset: ?, strides: [?
 // CHECK:         %[[s2:.*]] = subview {{%.*}}[{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32, #map{{.*}}> to memref<?x?xf32, #map{{.*}}>
 // CHECK:         %[[a0:.*]] = alloc({{%.*}}) : memref<?xi8>
 // CHECK:         %[[v0:.*]] = std.view %[[a0]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK:         %[[l0:.*]] = subview %[[v0]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK:         %[[l0:.*]] = subview %[[v0]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK-NOT:     %[[a1:.*]] = alloc({{%.*}}) : memref<?xi8>
 // CHECK-NOT:     %[[v1:.*]] = std.view %[[a1]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK-NOT:     %[[l0:.*]] = subview %[[v1]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK-NOT:     %[[l0:.*]] = subview %[[v1]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK-NOT:     %[[a2:.*]] = alloc({{%.*}}) : memref<?xi8>
 // CHECK-NOT:     %[[v2:.*]] = std.view %[[a2]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK-NOT:     %[[l0:.*]] = subview %[[v2]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK-NOT:     %[[l0:.*]] = subview %[[v2]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK:         linalg.copy(%[[s0]], %[[l0]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>
 // CHECK-NOT:     linalg.copy(%[[s1]], %[[l1]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>
 // CHECK-NOT:     linalg.copy(%[[s2]], %[[l2]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>^
@@ -337,7 +340,7 @@ func @aligned_promote_fill(%arg0: memref<?x?xf32, offset: ?, strides: [?, 1]>) {
 // CHECK:         %[[s0:.*]] = subview {{%.*}}[{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32, #map{{.*}}> to memref<?x?xf32, #map{{.*}}>
 // CHECK:         %[[a0:.*]] = alloc({{%.*}}) {alignment = 32 : i64} : memref<?xi8>
 // CHECK:         %[[v0:.*]] = std.view %[[a0]][{{.*}}][{{%.*}}, {{%.*}}] : memref<?xi8> to memref<?x?xf32>
-// CHECK:         %[[l0:.*]] = subview %[[v0]][{{%.*}}, {{%.*}}] [{{%.*}}, {{%.*}}] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D]]>
+// CHECK:         %[[l0:.*]] = subview %[[v0]][0, 0] [%{{.*}}, %{{.*}}] [1, 1] : memref<?x?xf32> to memref<?x?xf32, #[[$STRIDED_2D_u_1]]>
 // CHECK:         linalg.fill(%[[v0]], {{%.*}}) : memref<?x?xf32>, f32
 // CHECK:         linalg.copy(%[[s0]], %[[l0]]) : memref<?x?xf32, #map{{.*}}>, memref<?x?xf32, #map{{.*}}>
 // CHECK:         linalg.fill(%[[v0]], %[[cf]]) : memref<?x?xf32>, f32
diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
index a1552f11b64f..4757b8305c69 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir
@@ -8,7 +8,7 @@
 // LINALG-DAG: #[[$map_p4:.*]] = affine_map<()[s0] -> (s0 + 4)>
 // LINALG-DAG: #[[$map_p8:.*]] = affine_map<()[s0] -> (s0 + 8)>
 // LINALG-DAG: #[[$map_2d_stride_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
-// LINALG-DAG: #[[$map_2d_dynamic:.*]] = affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + s0 + d1 * s2)>
+// LINALG-DAG: #[[$map_2d_stride_8x1:.*]] = affine_map<(d0, d1)[s0] -> (d0 * 8 + s0 + d1)>
 // LINALG-DAG: #[[$bounds_map_4:.*]] = affine_map<(d0, d1, d2) -> (d0 - d1, 4)>
 // LINALG-DAG: #[[$bounds_map_8:.*]] = affine_map<(d0, d1, d2) -> (d0 - d1, 8)>
 
@@ -58,7 +58,6 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   // CHECK_SAME:   {masked = [false, false]} : memref<?x8xf32>, vector<4x8xf32>
 
   //  LINALG-DAG: %[[c0:.*]] = constant 0 : index
-  //  LINALG-DAG: %[[c1:.*]] = constant 1 : index
   //  LINALG-DAG: %[[c4:.*]] = constant 4 : index
   //  LINALG-DAG: %[[c8:.*]] = constant 8 : index
   //  LINALG-DAG: %[[cst:.*]] = constant 0.000000e+00 : f32
@@ -82,9 +81,9 @@ func @split_vector_transfer_read_2d(%A: memref<?x8xf32>, %i: index, %j: index) -
   //      LINALG:   %[[d0:.*]] = dim %[[A]], %[[c0]] : memref<?x8xf32>
   //      LINALG:   %[[sv0:.*]] = affine.min #[[$bounds_map_4]](%[[d0]], %[[i]], %[[c4]])
   //      LINALG:   %[[sv1:.*]] = affine.min #[[$bounds_map_8]](%[[c8]], %[[j]], %[[c8]])
-  //      LINALG:   %[[sv:.*]] = subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [%[[c1]], %[[c1]]]
-  // LINALG-SAME:     memref<?x8xf32> to memref<?x?xf32, #[[$map_2d_dynamic]]>
-  //      LINALG:   linalg.copy(%[[sv]], %[[alloc]]) : memref<?x?xf32, #[[$map_2d_dynamic]]>, memref<4x8xf32>
+  //      LINALG:   %[[sv:.*]] = subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [1, 1]
+  // LINALG-SAME:     memref<?x8xf32> to memref<?x?xf32, #[[$map_2d_stride_8x1]]>
+  //      LINALG:   linalg.copy(%[[sv]], %[[alloc]]) : memref<?x?xf32, #[[$map_2d_stride_8x1]]>, memref<4x8xf32>
   //      LINALG:   %[[yielded:.*]] = memref_cast %[[alloc]] :
   // LINALG-SAME:     memref<4x8xf32> to memref<?x8xf32>
   //      LINALG:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
@@ -150,7 +149,6 @@ func @split_vector_transfer_read_strided_2d(
   // CHECK-SAME:   memref<?x8xf32, #[[$map_2d_stride_1]]>, vector<4x8xf32>
 
   //  LINALG-DAG: %[[c0:.*]] = constant 0 : index
-  //  LINALG-DAG: %[[c1:.*]] = constant 1 : index
   //  LINALG-DAG: %[[c4:.*]] = constant 4 : index
   //  LINALG-DAG: %[[c7:.*]] = constant 7 : index
   //  LINALG-DAG: %[[c8:.*]] = constant 8 : index
@@ -176,9 +174,9 @@ func @split_vector_transfer_read_strided_2d(
   //      LINALG:   linalg.fill(%[[alloc]], %[[cst]]) : memref<4x8xf32>, f32
   //      LINALG:   %[[sv0:.*]] = affine.min #[[$bounds_map_4]](%[[c7]], %[[i]], %[[c4]])
   //      LINALG:   %[[sv1:.*]] = affine.min #[[$bounds_map_8]](%[[c8]], %[[j]], %[[c8]])
-  //      LINALG:   %[[sv:.*]] = subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [%[[c1]], %[[c1]]]
-  // LINALG-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x?xf32, #[[$map_2d_dynamic]]>
-  //      LINALG:   linalg.copy(%[[sv]], %[[alloc]]) : memref<?x?xf32, #[[$map_2d_dynamic]]>, memref<4x8xf32>
+  //      LINALG:   %[[sv:.*]] = subview %[[A]][%[[i]], %[[j]]] [%[[sv0]], %[[sv1]]] [1, 1]
+  // LINALG-SAME:     memref<7x8xf32, #[[$map_2d_stride_1]]> to memref<?x?xf32, #[[$map_2d_stride_1]]>
+  //      LINALG:   linalg.copy(%[[sv]], %[[alloc]]) : memref<?x?xf32, #[[$map_2d_stride_1]]>, memref<4x8xf32>
   //      LINALG:   %[[yielded:.*]] = memref_cast %[[alloc]] :
   // LINALG-SAME:     memref<4x8xf32> to memref<?x8xf32, #[[$map_2d_stride_1]]>
   //      LINALG:   scf.yield %[[yielded]], %[[c0]], %[[c0]] :
diff --git a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
index c2b4c7b9c821..a322b627756e 100644
--- a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
+#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
@@ -79,6 +80,12 @@ struct TestLinalgTransforms
       *this, "test-affine-min-scf-canonicalization-patterns",
       llvm::cl::desc("Test affine-min + scf canonicalization patterns."),
       llvm::cl::init(false)};
+  Option<bool> testTileAndPadPattern{
+      *this, "test-tile-and-pad-pattern",
+      llvm::cl::desc("Test tile and pad pattern"), llvm::cl::init(false)};
+  Option<bool> testHoistPadding2Levels{*this, "test-hoist-padding-2-level",
+                                       llvm::cl::desc("Test hoist padding"),
+                                       llvm::cl::init(false)};
 };
 } // end anonymous namespace
 
@@ -487,6 +494,27 @@ static void applyAffineMinSCFCanonicalizationPatterns(FuncOp funcOp) {
     applyOpPatternsAndFold(minOp, frozenPatterns);
   });
 }
+
+// For now, just assume it is the zero of type.
+// In the future, it should be the zero of type + op.
+static Value getNeutralOfLinalgOp(OpBuilder &b, Operation *op) {
+  auto t = op->getResult(0).getType().cast<ShapedType>().getElementType();
+  return b.create<ConstantOp>(op->getLoc(), t, b.getZeroAttr(t));
+}
+
+static void applyTileAndPadPattern(FuncOp funcOp) {
+  MLIRContext *context = funcOp.getContext();
+  OwningRewritePatternList tilingPattern;
+  auto linalgTilingOptions =
+      linalg::LinalgTilingOptions()
+          .setTileSizes({2, 3, 4})
+          .setPaddingValueComputationFunction(getNeutralOfLinalgOp);
+  tilingPattern.insert<linalg::LinalgTilingPattern<linalg::MatmulOp>>(
+      context, linalgTilingOptions,
+      linalg::LinalgMarker(Identifier::get("tile-and-pad", context)));
+  applyPatternsAndFoldGreedily(funcOp, std::move(tilingPattern));
+}
+
 /// Apply transformations specified as patterns.
 void TestLinalgTransforms::runOnFunction() {
   auto lambda = [&](void *) {
@@ -520,6 +548,13 @@ void TestLinalgTransforms::runOnFunction() {
     return applyLinalgToVectorPatterns(getFunction());
   if (testAffineMinSCFCanonicalizationPatterns)
     return applyAffineMinSCFCanonicalizationPatterns(getFunction());
+  if (testTileAndPadPattern)
+    return applyTileAndPadPattern(getFunction());
+  if (testHoistPadding2Levels) {
+    getFunction().walk([](linalg::SimplePadOp simplePadOp) {
+      linalg::hoistPaddingOnTensors(simplePadOp, 2);
+    });
+  }
 }
 
 namespace mlir {
diff --git a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp
index ad67fe95c77e..cc7bc42412f6 100644
--- a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp
+++ b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.cpp
@@ -28,26 +28,26 @@ DLWRAP(cuFuncGetAttribute, 3);
 DLWRAP(cuGetErrorString, 2);
 DLWRAP(cuLaunchKernel, 11);
 
-DLWRAP(cuMemAlloc_v2, 2);
-DLWRAP(cuMemcpyDtoDAsync_v2, 4);
+DLWRAP(cuMemAlloc, 2);
+DLWRAP(cuMemcpyDtoDAsync, 4);
 
-DLWRAP(cuMemcpyDtoH_v2, 3);
-DLWRAP(cuMemcpyDtoHAsync_v2, 4);
-DLWRAP(cuMemcpyHtoD_v2, 3);
-DLWRAP(cuMemcpyHtoDAsync_v2, 4);
+DLWRAP(cuMemcpyDtoH, 3);
+DLWRAP(cuMemcpyDtoHAsync, 4);
+DLWRAP(cuMemcpyHtoD, 3);
+DLWRAP(cuMemcpyHtoDAsync, 4);
 
-DLWRAP(cuMemFree_v2, 1);
+DLWRAP(cuMemFree, 1);
 DLWRAP(cuModuleGetFunction, 3);
-DLWRAP(cuModuleGetGlobal_v2, 4);
+DLWRAP(cuModuleGetGlobal, 4);
 
 DLWRAP(cuModuleUnload, 1);
 DLWRAP(cuStreamCreate, 2);
-DLWRAP(cuStreamDestroy_v2, 1);
+DLWRAP(cuStreamDestroy, 1);
 DLWRAP(cuStreamSynchronize, 1);
 DLWRAP(cuCtxSetCurrent, 1);
-DLWRAP(cuDevicePrimaryCtxRelease_v2, 1);
+DLWRAP(cuDevicePrimaryCtxRelease, 1);
 DLWRAP(cuDevicePrimaryCtxGetState, 3);
-DLWRAP(cuDevicePrimaryCtxSetFlags_v2, 2);
+DLWRAP(cuDevicePrimaryCtxSetFlags, 2);
 DLWRAP(cuDevicePrimaryCtxRetain, 2);
 DLWRAP(cuModuleLoadDataEx, 5);
 
diff --git a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h
index dd579a1f7490..832c26965144 100644
--- a/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h
+++ b/openmp/libomptarget/plugins/cuda/dynamic_cuda/cuda.h
@@ -48,6 +48,18 @@ typedef enum CUctx_flags_enum {
   CU_CTX_SCHED_MASK = 0x07,
 } CUctx_flags;
 
+#define cuMemFree cuMemFree_v2
+#define cuMemAlloc cuMemAlloc_v2
+#define cuMemcpyDtoH cuMemcpyDtoH_v2
+#define cuMemcpyHtoD cuMemcpyHtoD_v2
+#define cuStreamDestroy cuStreamDestroy_v2
+#define cuModuleGetGlobal cuModuleGetGlobal_v2
+#define cuMemcpyDtoHAsync cuMemcpyDtoHAsync_v2
+#define cuMemcpyDtoDAsync cuMemcpyDtoDAsync_v2
+#define cuMemcpyHtoDAsync cuMemcpyHtoDAsync_v2
+#define cuDevicePrimaryCtxRelease cuDevicePrimaryCtxRelease_v2
+#define cuDevicePrimaryCtxSetFlags cuDevicePrimaryCtxSetFlags_v2
+
 CUresult cuCtxGetDevice(CUdevice *);
 CUresult cuDeviceGet(CUdevice *, int);
 CUresult cuDeviceGetAttribute(int *, CUdevice_attribute, CUdevice);
@@ -60,26 +72,26 @@ CUresult cuLaunchKernel(CUfunction, unsigned, unsigned, unsigned, unsigned,
                         unsigned, unsigned, unsigned, CUstream, void **,
                         void **);
 
-CUresult cuMemAlloc_v2(CUdeviceptr *, size_t);
-CUresult cuMemcpyDtoDAsync_v2(CUdeviceptr, CUdeviceptr, size_t, CUstream);
+CUresult cuMemAlloc(CUdeviceptr *, size_t);
+CUresult cuMemcpyDtoDAsync(CUdeviceptr, CUdeviceptr, size_t, CUstream);
 
-CUresult cuMemcpyDtoH_v2(void *, CUdeviceptr, size_t);
-CUresult cuMemcpyDtoHAsync_v2(void *, CUdeviceptr, size_t, CUstream);
-CUresult cuMemcpyHtoD_v2(CUdeviceptr, const void *, size_t);
-CUresult cuMemcpyHtoDAsync_v2(CUdeviceptr, const void *, size_t, CUstream);
+CUresult cuMemcpyDtoH(void *, CUdeviceptr, size_t);
+CUresult cuMemcpyDtoHAsync(void *, CUdeviceptr, size_t, CUstream);
+CUresult cuMemcpyHtoD(CUdeviceptr, const void *, size_t);
+CUresult cuMemcpyHtoDAsync(CUdeviceptr, const void *, size_t, CUstream);
 
-CUresult cuMemFree_v2(CUdeviceptr);
+CUresult cuMemFree(CUdeviceptr);
 CUresult cuModuleGetFunction(CUfunction *, CUmodule, const char *);
-CUresult cuModuleGetGlobal_v2(CUdeviceptr *, size_t *, CUmodule, const char *);
+CUresult cuModuleGetGlobal(CUdeviceptr *, size_t *, CUmodule, const char *);
 
 CUresult cuModuleUnload(CUmodule);
 CUresult cuStreamCreate(CUstream *, unsigned);
-CUresult cuStreamDestroy_v2(CUstream);
+CUresult cuStreamDestroy(CUstream);
 CUresult cuStreamSynchronize(CUstream);
 CUresult cuCtxSetCurrent(CUcontext);
-CUresult cuDevicePrimaryCtxRelease_v2(CUdevice);
+CUresult cuDevicePrimaryCtxRelease(CUdevice);
 CUresult cuDevicePrimaryCtxGetState(CUdevice, unsigned *, int *);
-CUresult cuDevicePrimaryCtxSetFlags_v2(CUdevice, unsigned);
+CUresult cuDevicePrimaryCtxSetFlags(CUdevice, unsigned);
 CUresult cuDevicePrimaryCtxRetain(CUcontext *, CUdevice);
 CUresult cuModuleLoadDataEx(CUmodule *, const void *, unsigned, void *,
                             void **);
diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
index f83c9df920aa..e4ac1e0820e6 100644
--- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
@@ -110,8 +110,8 @@ bool checkResult(CUresult Err, const char *ErrMsg) {
 
 int memcpyDtoD(const void *SrcPtr, void *DstPtr, int64_t Size,
                CUstream Stream) {
-  CUresult Err = cuMemcpyDtoDAsync_v2((CUdeviceptr)DstPtr, (CUdeviceptr)SrcPtr,
-                                      Size, Stream);
+  CUresult Err =
+      cuMemcpyDtoDAsync((CUdeviceptr)DstPtr, (CUdeviceptr)SrcPtr, Size, Stream);
 
   if (Err != CUDA_SUCCESS) {
     REPORT("Error when copying data from device to device. Pointers: src "
@@ -207,8 +207,8 @@ class StreamManagerTy {
 
       for (CUstream &S : StreamPool[I]) {
         if (S)
-          checkResult(cuStreamDestroy_v2(S),
-                      "Error returned from cuStreamDestroy_v2\n");
+          checkResult(cuStreamDestroy(S),
+                      "Error returned from cuStreamDestroy\n");
       }
     }
   }
@@ -311,8 +311,8 @@ class DeviceRTLTy {
         return nullptr;
 
       CUdeviceptr DevicePtr;
-      Err = cuMemAlloc_v2(&DevicePtr, Size);
-      if (!checkResult(Err, "Error returned from cuMemAlloc_v2\n"))
+      Err = cuMemAlloc(&DevicePtr, Size);
+      if (!checkResult(Err, "Error returned from cuMemAlloc\n"))
         return nullptr;
 
       return (void *)DevicePtr;
@@ -323,8 +323,8 @@ class DeviceRTLTy {
       if (!checkResult(Err, "Error returned from cuCtxSetCurrent\n"))
         return OFFLOAD_FAIL;
 
-      Err = cuMemFree_v2((CUdeviceptr)TgtPtr);
-      if (!checkResult(Err, "Error returned from cuMemFree_v2\n"))
+      Err = cuMemFree((CUdeviceptr)TgtPtr);
+      if (!checkResult(Err, "Error returned from cuMemFree\n"))
         return OFFLOAD_FAIL;
 
       return OFFLOAD_SUCCESS;
@@ -466,8 +466,8 @@ class DeviceRTLTy {
         CUdevice Device;
         checkResult(cuCtxGetDevice(&Device),
                     "Error returned from cuCtxGetDevice\n");
-        checkResult(cuDevicePrimaryCtxRelease_v2(Device),
-                    "Error returned from cuDevicePrimaryCtxRelease_v2\n");
+        checkResult(cuDevicePrimaryCtxRelease(Device),
+                    "Error returned from cuDevicePrimaryCtxRelease\n");
       }
     }
   }
@@ -506,9 +506,8 @@ class DeviceRTLTy {
     } else {
       DP("The primary context is inactive, set its flags to "
          "CU_CTX_SCHED_BLOCKING_SYNC\n");
-      Err = cuDevicePrimaryCtxSetFlags_v2(Device, CU_CTX_SCHED_BLOCKING_SYNC);
-      if (!checkResult(Err,
-                       "Error returned from cuDevicePrimaryCtxSetFlags_v2\n"))
+      Err = cuDevicePrimaryCtxSetFlags(Device, CU_CTX_SCHED_BLOCKING_SYNC);
+      if (!checkResult(Err, "Error returned from cuDevicePrimaryCtxSetFlags\n"))
         return OFFLOAD_FAIL;
     }
 
@@ -657,7 +656,7 @@ class DeviceRTLTy {
         __tgt_offload_entry Entry = *E;
         CUdeviceptr CUPtr;
         size_t CUSize;
-        Err = cuModuleGetGlobal_v2(&CUPtr, &CUSize, Module, E->name);
+        Err = cuModuleGetGlobal(&CUPtr, &CUSize, Module, E->name);
         // We keep this style here because we need the name
         if (Err != CUDA_SUCCESS) {
           REPORT("Loading global '%s' Failed\n", E->name);
@@ -689,7 +688,7 @@ class DeviceRTLTy {
           // If unified memory is present any target link or to variables
           // can access host addresses directly. There is no longer a
           // need for device copies.
-          cuMemcpyHtoD_v2(CUPtr, E->addr, sizeof(void *));
+          cuMemcpyHtoD(CUPtr, E->addr, sizeof(void *));
           DP("Copy linked variable host address (" DPxMOD
              ") to device address (" DPxMOD ")\n",
              DPxPTR(*((void **)E->addr)), DPxPTR(CUPtr));
@@ -720,7 +719,7 @@ class DeviceRTLTy {
 
       CUdeviceptr ExecModePtr;
       size_t CUSize;
-      Err = cuModuleGetGlobal_v2(&ExecModePtr, &CUSize, Module, ExecModeName);
+      Err = cuModuleGetGlobal(&ExecModePtr, &CUSize, Module, ExecModeName);
       if (Err == CUDA_SUCCESS) {
         if (CUSize != sizeof(int8_t)) {
           DP("Loading global exec_mode '%s' - size mismatch (%zd != %zd)\n",
@@ -728,7 +727,7 @@ class DeviceRTLTy {
           return nullptr;
         }
 
-        Err = cuMemcpyDtoH_v2(&ExecModeVal, ExecModePtr, CUSize);
+        Err = cuMemcpyDtoH(&ExecModeVal, ExecModePtr, CUSize);
         if (Err != CUDA_SUCCESS) {
           REPORT("Error when copying data from device to host. Pointers: "
                  "host = " DPxMOD ", device = " DPxMOD ", size = %zd\n",
@@ -769,7 +768,7 @@ class DeviceRTLTy {
       CUdeviceptr DeviceEnvPtr;
       size_t CUSize;
 
-      Err = cuModuleGetGlobal_v2(&DeviceEnvPtr, &CUSize, Module, DeviceEnvName);
+      Err = cuModuleGetGlobal(&DeviceEnvPtr, &CUSize, Module, DeviceEnvName);
       if (Err == CUDA_SUCCESS) {
         if (CUSize != sizeof(DeviceEnv)) {
           REPORT(
@@ -779,7 +778,7 @@ class DeviceRTLTy {
           return nullptr;
         }
 
-        Err = cuMemcpyHtoD_v2(DeviceEnvPtr, &DeviceEnv, CUSize);
+        Err = cuMemcpyHtoD(DeviceEnvPtr, &DeviceEnv, CUSize);
         if (Err != CUDA_SUCCESS) {
           REPORT("Error when copying data from host to device. Pointers: "
                  "host = " DPxMOD ", device = " DPxMOD ", size = %zu\n",
@@ -817,7 +816,7 @@ class DeviceRTLTy {
 
     CUstream Stream = getStream(DeviceId, AsyncInfoPtr);
 
-    Err = cuMemcpyHtoDAsync_v2((CUdeviceptr)TgtPtr, HstPtr, Size, Stream);
+    Err = cuMemcpyHtoDAsync((CUdeviceptr)TgtPtr, HstPtr, Size, Stream);
     if (Err != CUDA_SUCCESS) {
       REPORT("Error when copying data from host to device. Pointers: host "
              "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 "\n",
@@ -839,7 +838,7 @@ class DeviceRTLTy {
 
     CUstream Stream = getStream(DeviceId, AsyncInfoPtr);
 
-    Err = cuMemcpyDtoHAsync_v2(HstPtr, (CUdeviceptr)TgtPtr, Size, Stream);
+    Err = cuMemcpyDtoHAsync(HstPtr, (CUdeviceptr)TgtPtr, Size, Stream);
     if (Err != CUDA_SUCCESS) {
       REPORT("Error when copying data from device to host. Pointers: host "
              "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 "\n",