diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index 3d2da01f2c856..4f1233ff592e0 100644
--- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -113,7 +113,7 @@ template <> struct ilist_traits<MachineInstr> {
   MachineBasicBlock *Parent;
 
   using instr_iterator =
-      simple_ilist<MachineInstr, ilist_sentinel_tracking<true>>::iterator;
+      simple_ilist<MachineInstr, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>::iterator;
 
 public:
   LLVM_ABI void addNodeToList(MachineInstr *N);
@@ -144,7 +144,7 @@ class MachineBasicBlock
   };
 
 private:
-  using Instructions = ilist<MachineInstr, ilist_sentinel_tracking<true>>;
+  using Instructions = ilist<MachineInstr, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>;
 
   const BasicBlock *BB;
   int Number;
@@ -1314,6 +1314,16 @@ class MachineBasicBlock
   /// unless you know what you're doing, because it doesn't update Pred's
   /// successors list. Use Pred->removeSuccessor instead.
   void removePredecessor(MachineBasicBlock *Pred);
+
+public:
+  DbgMachineMarker *getNextMarker(MachineInstr *I);
+  DbgMachineMarker *getMarker(iterator It);
+  void setTrailingDbgRecords(DbgMachineMarker *M);
+  DbgMachineMarker *getTrailingDbgRecords();
+  void deleteTrailingDbgRecords();
+  void flushTerminatorDbgRecords();
+  DbgMachineMarker *createMarker(MachineInstr *I);
+  DbgMachineMarker *createMarker(iterator I);
 };
 
 LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const MachineBasicBlock &MBB);
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index e5958ecc2432a..fc3e15e7b8e11 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -656,6 +656,7 @@ class LLVM_ABI MachineFunction {
   /// enough information for every DBG_INSTR_REF to point at an instruction
   /// (or DBG_PHI).
   void finalizeDebugInstrRefs();
+  std::optional<std::pair<unsigned, unsigned>> finalizeDebugInstrRefRef(Register R, DenseMap<Register, DebugInstrOperandPair> &ArgDbgPHIs);
 
   /// Determine whether, in the current machine configuration, we should use
   /// instruction referencing or not.
@@ -1469,6 +1470,16 @@ class LLVM_ABI MachineFunction {
   unsigned getNewDebugInstrNum() {
     return ++DebugInstrNumberingCount;
   }
+
+  SmallDenseMap<MachineBasicBlock *, DbgMachineMarker *> MachineTrailingDbgRecords;
+  void setTrailingDbgRecords(MachineBasicBlock *B, DbgMachineMarker *M) {
+    assert(!MachineTrailingDbgRecords.count(B));
+    MachineTrailingDbgRecords[B] = M;
+  }
+  DbgMachineMarker *getTrailingDbgRecords(MachineBasicBlock *B) {
+    return MachineTrailingDbgRecords.lookup(B);
+  }
+  void deleteTrailingDbgRecords(MachineBasicBlock *B) { MachineTrailingDbgRecords.erase(B); }
 };
 
 //===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 94d04b82666be..7bf28325c0a48 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -59,6 +59,11 @@ class StringRef;
 class TargetInstrInfo;
 class TargetRegisterClass;
 class TargetRegisterInfo;
+class DbgMachineMarker;
+class DbgMachineRecord;
+
+LLVM_ABI iterator_range<simple_ilist<DbgMachineRecord>::iterator>
+getDbgRecordRange(DbgMachineMarker *);
 
 //===----------------------------------------------------------------------===//
 /// Representation of each machine instruction.
@@ -69,9 +74,12 @@ class TargetRegisterInfo;
 ///
 class MachineInstr
     : public ilist_node_with_parent<MachineInstr, MachineBasicBlock,
+                                    ilist_iterator_bits<true>,
                                     ilist_sentinel_tracking<true>> {
 public:
   using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator;
+  using instr_iterator =
+      simple_ilist<MachineInstr, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>::iterator;
 
   /// Flags to specify different kinds of comments to output in
   /// assembly code.  These flags carry semantic information not
@@ -304,6 +312,8 @@ class MachineInstr
 
   DebugLoc DbgLoc; // Source line information.
 
+private:
+
   /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values
   /// defined by this instruction.
   unsigned DebugInstrNum;
@@ -359,6 +369,7 @@ class MachineInstr
 
   /// Move the instruction before \p MovePos.
   LLVM_ABI void moveBefore(MachineInstr *MovePos);
+  LLVM_ABI void moveBefore(instr_iterator MovePos);
 
   /// Return the function that contains the basic block that this instruction
   /// belongs to.
@@ -1388,6 +1399,11 @@ class MachineInstr
     return false;
   }
 
+  /// Return a range over the DbgRecords attached to this instruction.
+  iterator_range<simple_ilist<DbgMachineRecord>::iterator> getDbgRecordRange() const {
+    return llvm::getDbgRecordRange(DebugMarker);
+  }
+
   bool isJumpTableDebugInfo() const {
     return getOpcode() == TargetOpcode::JUMP_TABLE_DEBUG_INFO;
   }
@@ -2079,6 +2095,16 @@ class MachineInstr
                     MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol,
                     MDNode *HeapAllocMarker, MDNode *PCSections,
                     uint32_t CFIType, MDNode *MMRAs);
+
+public:
+  DbgMachineMarker *DebugMarker = nullptr;
+  void handleMarkerRemoval();
+  /// Transfer any DbgRecords on the position \p It onto this instruction,
+  /// by simply adopting the sequence of DbgRecords (which is efficient) if
+  /// possible, by merging two sequences otherwise.
+  LLVM_ABI void adoptDbgRecords(MachineBasicBlock *BB, instr_iterator It,
+                                bool InsertAtHead);
+
 };
 
 /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the
diff --git a/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h b/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
index 250cb0d78a68f..fe3847111c0aa 100644
--- a/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
+++ b/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
@@ -23,25 +23,25 @@ namespace llvm {
 
 template <class T, bool IsReverse> struct MachineInstrBundleIteratorTraits;
 template <class T> struct MachineInstrBundleIteratorTraits<T, false> {
-  using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>;
+  using list_type = simple_ilist<T, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>;
   using instr_iterator = typename list_type::iterator;
   using nonconst_instr_iterator = typename list_type::iterator;
   using const_instr_iterator = typename list_type::const_iterator;
 };
 template <class T> struct MachineInstrBundleIteratorTraits<T, true> {
-  using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>;
+  using list_type = simple_ilist<T, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>;
   using instr_iterator = typename list_type::reverse_iterator;
   using nonconst_instr_iterator = typename list_type::reverse_iterator;
   using const_instr_iterator = typename list_type::const_reverse_iterator;
 };
 template <class T> struct MachineInstrBundleIteratorTraits<const T, false> {
-  using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>;
+  using list_type = simple_ilist<T, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>;
   using instr_iterator = typename list_type::const_iterator;
   using nonconst_instr_iterator = typename list_type::iterator;
   using const_instr_iterator = typename list_type::const_iterator;
 };
 template <class T> struct MachineInstrBundleIteratorTraits<const T, true> {
-  using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>;
+  using list_type = simple_ilist<T, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>>;
   using instr_iterator = typename list_type::const_reverse_iterator;
   using nonconst_instr_iterator = typename list_type::reverse_iterator;
   using const_instr_iterator = typename list_type::const_reverse_iterator;
diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h
index e0292c2b8d2d2..186442a8fc0e9 100644
--- a/llvm/include/llvm/IR/DebugProgramInstruction.h
+++ b/llvm/include/llvm/IR/DebugProgramInstruction.h
@@ -57,6 +57,8 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 
+#include "llvm/CodeGen/MachineInstr.h"
+
 namespace llvm {
 
 class Instruction;
@@ -70,6 +72,8 @@ class DIAssignID;
 class DbgMarker;
 class DbgVariableRecord;
 class raw_ostream;
+class DbgMachineRecord;
+class DbgMachineMarker;
 
 /// A typed tracking MDNode reference that does not require a definition for its
 /// parameter type. Necessary to avoid including DebugInfoMetadata.h, which has
@@ -136,10 +140,15 @@ extern template class LLVM_TEMPLATE_ABI DbgRecordParamRef<DILocalVariable>;
 ///   isIdenticalToWhenDefined
 ///   both print methods
 ///   createDebugIntrinsic
-class DbgRecord : public ilist_node<DbgRecord> {
+template <typename InstT, typename BlockT, typename FuncT, typename MarkerT, typename CRTP>
+class DbgRecordBase : public ilist_node<CRTP> {
 public:
-  /// Marker that this DbgRecord is linked into.
-  DbgMarker *Marker = nullptr;
+  /// Marker that this DbgRecordBase is linked into.
+  MarkerT *Marker = nullptr;
+  using self_iterator = typename ilist_node<CRTP>::self_iterator;
+  using const_self_iterator = typename simple_ilist<CRTP>::const_iterator;
+  using BaseT = ilist_node<CRTP>;
+
   /// Subclass discriminator.
   enum Kind : uint8_t { ValueKind, LabelKind };
 
@@ -148,83 +157,138 @@ class DbgRecord : public ilist_node<DbgRecord> {
   Kind RecordKind; ///< Subclass discriminator.
 
 public:
-  DbgRecord(Kind RecordKind, DebugLoc DL)
+  DbgRecordBase(DebugLoc DL, Kind RecordKind)
       : DbgLoc(DL), RecordKind(RecordKind) {}
 
-  /// Methods that dispatch to subclass implementations. These need to be
-  /// manually updated when a new subclass is added.
-  ///@{
-  LLVM_ABI void deleteRecord();
-  LLVM_ABI DbgRecord *clone() const;
-  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
-  LLVM_ABI void print(raw_ostream &O, ModuleSlotTracker &MST,
-                      bool IsForDebug) const;
-  LLVM_ABI bool isIdenticalToWhenDefined(const DbgRecord &R) const;
-  /// Convert this DbgRecord back into an appropriate llvm.dbg.* intrinsic.
-  /// \p InsertBefore Optional position to insert this intrinsic.
-  /// \returns A new llvm.dbg.* intrinsic representiung this DbgRecord.
-  LLVM_ABI DbgInfoIntrinsic *
-  createDebugIntrinsic(Module *M, Instruction *InsertBefore) const;
-  ///@}
-
-  /// Same as isIdenticalToWhenDefined but checks DebugLoc too.
-  LLVM_ABI bool isEquivalentTo(const DbgRecord &R) const;
-
-  Kind getRecordKind() const { return RecordKind; }
-
-  void setMarker(DbgMarker *M) { Marker = M; }
+  void setMarker(MarkerT *M) { Marker = M; }
 
-  DbgMarker *getMarker() { return Marker; }
-  const DbgMarker *getMarker() const { return Marker; }
+  MarkerT *getMarker() { return Marker; }
+  const MarkerT *getMarker() const { return Marker; }
 
-  LLVM_ABI BasicBlock *getBlock();
-  LLVM_ABI const BasicBlock *getBlock() const;
+  BlockT *getBlock() {
+    return getInstruction()->getParent();
+  }
+  const BlockT *getBlock() const {
+    return getInstruction()->getParent();
+  }
 
-  LLVM_ABI Function *getFunction();
-  LLVM_ABI const Function *getFunction() const;
+  FuncT *getFunction() {
+    return getBlock()->getParent();
+  }
+  const FuncT *getFunction() const {
+    return getBlock()->getParent();
+  }
 
-  LLVM_ABI Module *getModule();
-  LLVM_ABI const Module *getModule() const;
+  Module *getModule() {
+    return getFunction()->getParent();
+  }
+  const Module *getModule() const {
+    return getFunction()->getParent();
+  }
 
-  LLVM_ABI LLVMContext &getContext();
-  LLVM_ABI const LLVMContext &getContext() const;
+  LLVMContext &getContext() {
+    return getBlock()->getContext();
+  }
+  const LLVMContext &getContext() const {
+    return getBlock()->getContext();
+  }
 
-  LLVM_ABI const Instruction *getInstruction() const;
-  LLVM_ABI const BasicBlock *getParent() const;
-  LLVM_ABI BasicBlock *getParent();
+  const InstT *getInstruction() const;
+  InstT *getInstruction();
 
-  LLVM_ABI void removeFromParent();
-  LLVM_ABI void eraseFromParent();
+  const BlockT *getParent() const {
+    return getBlock();
+  }
+  BlockT *getParent() {
+    return getBlock();
+  }
 
-  DbgRecord *getNextNode() { return &*std::next(getIterator()); }
-  DbgRecord *getPrevNode() { return &*std::prev(getIterator()); }
+  void removeFromParent() {
+    getMarker()->StoredDbgRecords.erase(BaseT::getIterator());
+    Marker = nullptr;
+  }
 
-  // Some generic lambdas supporting intrinsic-based debug-info mean we need
-  // to support both iterator and instruction position based insertion.
-  LLVM_ABI void insertBefore(DbgRecord *InsertBefore);
-  LLVM_ABI void insertAfter(DbgRecord *InsertAfter);
-  LLVM_ABI void moveBefore(DbgRecord *MoveBefore);
-  LLVM_ABI void moveAfter(DbgRecord *MoveAfter);
+  DbgRecordBase *getNextNode() { return &*std::next(ilist_node<CRTP>::getIterator()); }
+  DbgRecordBase *getPrevNode() { return &*std::prev(ilist_node<CRTP>::getIterator()); }
+
+  void insertBefore(DbgRecordBase *InsertBefore) {
+    assert(!getMarker() &&
+           "Cannot insert a DbgRecord that is already has a DbgMarker!");
+    assert(InsertBefore->getMarker() &&
+           "Cannot insert a DbgRecord before a DbgRecord that does not have a "
+           "DbgMarker!");
+    CRTP *Derived = static_cast<CRTP*>(this);
+    CRTP *DerivedBefore = static_cast<CRTP*>(InsertBefore);
+    InsertBefore->getMarker()->insertDbgRecord(Derived, DerivedBefore);
+  }
+  void insertAfter(DbgRecordBase *InsertAfter) {
+    assert(!getMarker() &&
+           "Cannot insert a DbgRecord that is already has a DbgMarker!");
+    assert(InsertAfter->getMarker() &&
+           "Cannot insert a DbgRecord after a DbgRecord that does not have a "
+           "DbgMarker!");
+    CRTP *Derived = static_cast<CRTP*>(this);
+    CRTP *DerivedAfter = static_cast<CRTP*>(InsertAfter);
+    InsertAfter->getMarker()->insertDbgRecordAfter(Derived, DerivedAfter);
+  }
 
-  LLVM_ABI void insertBefore(self_iterator InsertBefore);
-  LLVM_ABI void insertAfter(self_iterator InsertAfter);
-  LLVM_ABI void moveBefore(self_iterator MoveBefore);
-  LLVM_ABI void moveAfter(self_iterator MoveAfter);
+  void moveBefore(DbgRecordBase *MoveBefore) {
+    assert(getMarker() &&
+           "Canot move a DbgRecord that does not currently have a DbgMarker!");
+    removeFromParent();
+    insertBefore(MoveBefore);
+  }
+  void moveAfter(DbgRecordBase *MoveAfter) {
+    assert(getMarker() &&
+           "Canot move a DbgRecord that does not currently have a DbgMarker!");
+    removeFromParent();
+    insertAfter(MoveAfter);
+  }
 
   DebugLoc getDebugLoc() const { return DbgLoc; }
   void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); }
 
-  LLVM_ABI void dump() const;
-
-  using self_iterator = simple_ilist<DbgRecord>::iterator;
-  using const_self_iterator = simple_ilist<DbgRecord>::const_iterator;
+  Kind getRecordKind() const { return RecordKind; }
 
 protected:
   /// Similarly to Value, we avoid paying the cost of a vtable
   /// by protecting the dtor and having deleteRecord dispatch
   /// cleanup.
   /// Use deleteRecord to delete a generic record.
-  ~DbgRecord() = default;
+  ~DbgRecordBase() = default;
+};
+
+class DbgRecord : public DbgRecordBase<Instruction, BasicBlock, Function, DbgMarker, DbgRecord> {
+public:
+  using BaseT = DbgRecordBase<Instruction, BasicBlock, Function, DbgMarker, DbgRecord>;
+  using self_iterator = typename BaseT::self_iterator;
+
+public:
+  DbgRecord(Kind RecordKind, DebugLoc DL)
+      : DbgRecordBase(DL, RecordKind) { }
+
+  LLVM_ABI void dump() const;
+  LLVM_ABI void deleteRecord();
+  LLVM_ABI void print(raw_ostream &O, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+
+  /// Methods that dispatch to subclass implementations. These need to be
+  /// manually updated when a new subclass is added.
+  ///@{
+  LLVM_ABI DbgRecord *clone() const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI bool isIdenticalToWhenDefined(const DbgRecord &R) const;
+  /// Convert this DbgRecord back into an appropriate llvm.dbg.* intrinsic.
+  /// \p InsertBefore Optional position to insert this intrinsic.
+  /// \returns A new llvm.dbg.* intrinsic representiung this DbgRecord.
+  LLVM_ABI DbgInfoIntrinsic *
+  createDebugIntrinsic(Module *M, Instruction *InsertBefore) const;
+  ///@}
+
+  void eraseFromParent() {
+    removeFromParent();
+    deleteRecord();
+  }
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const DbgRecord &R) {
@@ -566,80 +630,181 @@ filterDbgVars(iterator_range<simple_ilist<DbgRecord>::iterator> R) {
       [](DbgRecord &E) { return std::ref(cast<DbgVariableRecord>(E)); });
 }
 
-/// Per-instruction record of debug-info. If an Instruction is the position of
-/// some debugging information, it points at a DbgMarker storing that info. Each
-/// marker points back at the instruction that owns it. Various utilities are
-/// provided for manipulating the DbgRecords contained within this marker.
-///
-/// This class has a rough surface area, because it's needed to preserve the
-/// one arefact that we can't yet eliminate from the intrinsic / dbg.value
-/// debug-info design: the order of records is significant, and duplicates can
-/// exist. Thus, if one has a run of debug-info records such as:
-///    dbg.value(...
-///    %foo = barinst
-///    dbg.value(...
-/// and remove barinst, then the dbg.values must be preserved in the correct
-/// order. Hence, the use of iterators to select positions to insert things
-/// into, or the occasional InsertAtHead parameter indicating that new records
-/// should go at the start of the list.
-///
-/// There are only five or six places in LLVM that truly rely on this ordering,
-/// which we can improve in the future. Additionally, many improvements in the
-/// way that debug-info is stored can be achieved in this class, at a future
-/// date.
-class DbgMarker {
+class DbgMachineRecord : public DbgRecordBase<MachineInstr, MachineBasicBlock, MachineFunction, DbgMachineMarker, DbgMachineRecord> {
 public:
-  DbgMarker() {}
+  using BaseT = DbgRecordBase<MachineInstr, MachineBasicBlock, MachineFunction, DbgMachineMarker, DbgMachineRecord>;
+  using self_iterator = typename BaseT::self_iterator;
+
+public:
+  DbgMachineRecord(Kind RecordKind, DebugLoc DL)
+      : DbgRecordBase(DL, RecordKind) { }
+
+  LLVM_ABI void dump() const;
+  LLVM_ABI void deleteRecord();
+  LLVM_ABI void print(raw_ostream &O, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+
+  /// Methods that dispatch to subclass implementations. These need to be
+  /// manually updated when a new subclass is added.
+  ///@{
+  LLVM_ABI DbgMachineRecord *clone() const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI bool isIdenticalToWhenDefined(const DbgMachineRecord &R) const;
+  LLVM_ABI MachineInstr *
+  createDebugInstr(MachineInstr *InsertBefore) const;
+  ///@}
+
+  void eraseFromParent() {
+    removeFromParent();
+    deleteRecord();
+  }
+};
+
+/// XXX docs
+class DbgMachineLabelRecord : public DbgMachineRecord {
+  DbgRecordParamRef<DILabel> Label;
+
+  /// This constructor intentionally left private, so that it is only called via
+  /// "createUnresolvedDbgLabelRecord", which clearly expresses that it is for
+  /// parsing only.
+  DbgMachineLabelRecord(MDNode *Label, MDNode *DL);
+
+public:
+  LLVM_ABI DbgMachineLabelRecord(DILabel *Label, DebugLoc DL);
+
+  /// For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved
+  /// MDNodes. Trying to access the resulting DbgLabelRecord's fields before
+  /// they are resolved, or if they resolve to the wrong type, will result in a
+  /// crash.
+  LLVM_ABI static DbgMachineLabelRecord *createUnresolvedDbgMachineLabelRecord(MDNode *Label,
+                                                                 MDNode *DL);
+
+  LLVM_ABI DbgMachineLabelRecord *clone() const;
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+  LLVM_ABI MachineInstr *createMachineInstr(Module *M,
+                                              Instruction *InsertBefore) const;
+
+  void setLabel(DILabel *NewLabel) { Label = NewLabel; }
+  DILabel *getLabel() const { return Label.get(); }
+  MDNode *getRawLabel() const { return Label.getAsMDNode(); };
+
+  /// Support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(const DbgMachineRecord *E) {
+    return E->getRecordKind() == LabelKind;
+  }
+};
+
+class DbgMachineVariableRecord : public DbgMachineRecord {
+public:
+  enum class MachineLocationType : uint8_t {
+    Value, // i.e. DBG_VALUE,
+    Ref,   // i.e. DBG_INSTR_REF
+    PHI    // i.e. DBG_PHI. I suspect we can't fly with this?
+  };
+  MachineLocationType Type;
+
+  DbgRecordParamRef<DILocalVariable> Variable;
+  DbgRecordParamRef<DIExpression> Expression;
+
+public:
+  LLVM_ABI DbgMachineVariableRecord(DILocalVariable *Var, DIExpression *Expr, const DILocation *DILoc);
+
+  // XXX private parsing?
+
+  // Correctness before memory usage: Register for DBG_VALUE, DBG_PHI,
+  // Vector for DBG_INSTR_REFs.
+  Register Reg;
+  SmallVector<std::pair<unsigned, unsigned>, 1> Refs;
+
+  LLVM_ABI static DbgMachineVariableRecord *
+  createDMVRValue(Register R, DILocalVariable *Variable,
+                  DIExpression *Expression, const DILocation *DI);
+
+  LLVM_ABI static DbgMachineVariableRecord *
+  createDMVRRef(ArrayRef<std::pair<unsigned, unsigned>>,
+                          DILocalVariable *DV,
+                          DIExpression *Expr, const DILocation *DI);
+
+  LLVM_ABI static DbgMachineVariableRecord *
+  createDMVRPHI(Register R, DILocalVariable *Variable,
+                  DIExpression *Expression, const DILocation *DI);
+
+  // XXX -- storage for _arrays_ of variadic DBG_INSTR_REFs?
+
+  bool isValue() const { return Type == MachineLocationType::Value; }
+  bool isRef() const { return Type == MachineLocationType::Ref; }
+  bool isPHI() const { return Type == MachineLocationType::PHI; }
+
+  void setVariable(DILocalVariable *NewVar) { Variable = NewVar; }
+  DILocalVariable *getVariable() const { return Variable.get(); };
+  MDNode *getRawVariable() const { return Variable.getAsMDNode(); }
+
+  void setExpression(DIExpression *NewExpr) { Expression = NewExpr; }
+  DIExpression *getExpression() const { return Expression.get(); }
+  MDNode *getRawExpression() const { return Expression.getAsMDNode(); }
+
+#if 0
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+#endif
+
+  /// Support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(const DbgMachineRecord *E) {
+    return E->getRecordKind() == ValueKind;
+  }
+};
+
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+class DbgMarkerBase {
+public:
+  DbgMarkerBase() {}
   /// Link back to the Instruction that owns this marker. Can be null during
   /// operations that move a marker from one instruction to another.
-  Instruction *MarkedInstr = nullptr;
+  InstT *MarkedInstr = nullptr;
 
-  /// List of DbgRecords, the non-instruction equivalent of llvm.dbg.*
-  /// intrinsics. There is a one-to-one relationship between each debug
-  /// intrinsic in a block and each DbgRecord once the representation has been
-  /// converted, and the ordering is meaningful in the same way.
-  simple_ilist<DbgRecord> StoredDbgRecords;
+  /// List of DbgRecords (or similar), the non-instruction equivalent of
+  /// llvm.dbg.* intrinsics. There is a one-to-one relationship between each
+  /// debug intrinsic in a block and each DbgRecord once the representation has
+  /// been converted, and the ordering is meaningful in the same way.
+  simple_ilist<RecT> StoredDbgRecords;
   bool empty() const { return StoredDbgRecords.empty(); }
 
-  LLVM_ABI const BasicBlock *getParent() const;
-  LLVM_ABI BasicBlock *getParent();
+  LLVM_ABI const BlockT *getParent() const;
+  LLVM_ABI BlockT *getParent();
 
   /// Handle the removal of a marker: the position of debug-info has gone away,
   /// but the stored debug records should not. Drop them onto the next
   /// instruction, or otherwise work out what to do with them.
   LLVM_ABI void removeMarker();
-  LLVM_ABI void dump() const;
 
   LLVM_ABI void removeFromParent();
   LLVM_ABI void eraseFromParent();
 
-  /// Implement operator<< on DbgMarker.
-  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
-  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
-                      bool IsForDebug) const;
-
   /// Produce a range over all the DbgRecords in this Marker.
-  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::iterator>
+  LLVM_ABI iterator_range<typename simple_ilist<RecT>::iterator>
   getDbgRecordRange();
-  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::const_iterator>
+  LLVM_ABI iterator_range<typename simple_ilist<RecT>::const_iterator>
   getDbgRecordRange() const;
   /// Transfer any DbgRecords from \p Src into this DbgMarker. If \p
   /// InsertAtHead is true, place them before existing DbgRecords, otherwise
   /// afterwards.
-  LLVM_ABI void absorbDebugValues(DbgMarker &Src, bool InsertAtHead);
+  LLVM_ABI void absorbDebugValues(DbgMarkerBase &Src, bool InsertAtHead);
   /// Transfer the DbgRecords in \p Range from \p Src into this DbgMarker. If
   /// \p InsertAtHead is true, place them before existing DbgRecords, otherwise
   // afterwards.
   LLVM_ABI void
-  absorbDebugValues(iterator_range<DbgRecord::self_iterator> Range,
-                    DbgMarker &Src, bool InsertAtHead);
+  absorbDebugValues(iterator_range<typename RecT::self_iterator> Range,
+                    DbgMarkerBase &Src, bool InsertAtHead);
   /// Insert a DbgRecord into this DbgMarker, at the end of the list. If
   /// \p InsertAtHead is true, at the start.
-  LLVM_ABI void insertDbgRecord(DbgRecord *New, bool InsertAtHead);
+  LLVM_ABI void insertDbgRecord(RecT *New, bool InsertAtHead);
   /// Insert a DbgRecord prior to a DbgRecord contained within this marker.
-  LLVM_ABI void insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore);
+  LLVM_ABI void insertDbgRecord(RecT *New, RecT *InsertBefore);
   /// Insert a DbgRecord after a DbgRecord contained within this marker.
-  LLVM_ABI void insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter);
+  LLVM_ABI void insertDbgRecordAfter(RecT *New, RecT *InsertAfter);
   /// Clone all DbgMarkers from \p From into this marker. There are numerous
   /// options to customise the source/destination, due to gnarliness, see class
   /// comment.
@@ -648,16 +813,16 @@ class DbgMarker {
   /// \p InsertAtHead Place the cloned DbgRecords at the start of
   /// StoredDbgRecords
   /// \returns Range over all the newly cloned DbgRecords
-  LLVM_ABI iterator_range<simple_ilist<DbgRecord>::iterator>
-  cloneDebugInfoFrom(DbgMarker *From,
-                     std::optional<simple_ilist<DbgRecord>::iterator> FromHere,
+  LLVM_ABI iterator_range<typename simple_ilist<RecT>::iterator>
+  cloneDebugInfoFrom(DbgMarkerBase *From,
+                     std::optional<typename simple_ilist<RecT>::iterator> FromHere,
                      bool InsertAtHead = false);
   /// Erase all DbgRecords in this DbgMarker.
   LLVM_ABI void dropDbgRecords();
   /// Erase a single DbgRecord from this marker. In an ideal future, we would
   /// never erase an assignment in this way, but it's the equivalent to
   /// erasing a debug intrinsic from a block.
-  LLVM_ABI void dropOneDbgRecord(DbgRecord *DR);
+  LLVM_ABI void dropOneDbgRecord(RecT *DR);
 
   /// We generally act like all llvm Instructions have a range of DbgRecords
   /// attached to them, but in reality sometimes we don't allocate the DbgMarker
@@ -666,14 +831,60 @@ class DbgMarker {
   /// static markers range instead. This will bite us if someone tries to insert
   /// a DbgRecord in that range, but they should be using the Official (TM) API
   /// for that.
-  LLVM_ABI static DbgMarker EmptyDbgMarker;
-  static iterator_range<simple_ilist<DbgRecord>::iterator>
+  LLVM_ABI static DbgMarkerBase<RecT, InstT, BlockT, CRTP> EmptyDbgMarker;
+  static iterator_range<typename simple_ilist<RecT>::iterator>
   getEmptyDbgRecordRange() {
     return make_range(EmptyDbgMarker.StoredDbgRecords.end(),
                       EmptyDbgMarker.StoredDbgRecords.end());
   }
 };
 
+/// Per-instruction record of debug-info. If an Instruction is the position of
+/// some debugging information, it points at a DbgMarker storing that info. Each
+/// marker points back at the instruction that owns it. Various utilities are
+/// provided for manipulating the DbgRecords contained within this marker.
+///
+/// This class has a rough surface area, because it's needed to preserve the
+/// one arefact that we can't yet eliminate from the intrinsic / dbg.value
+/// debug-info design: the order of records is significant, and duplicates can
+/// exist. Thus, if one has a run of debug-info records such as:
+///    dbg.value(...
+///    %foo = barinst
+///    dbg.value(...
+/// and remove barinst, then the dbg.values must be preserved in the correct
+/// order. Hence, the use of iterators to select positions to insert things
+/// into, or the occasional InsertAtHead parameter indicating that new records
+/// should go at the start of the list.
+///
+/// There are only five or six places in LLVM that truly rely on this ordering,
+/// which we can improve in the future. Additionally, many improvements in the
+/// way that debug-info is stored can be achieved in this class, at a future
+/// date.
+class DbgMarker : public DbgMarkerBase<DbgRecord, Instruction, BasicBlock, DbgMarker> {
+public:
+  DbgMarker() : DbgMarkerBase() {}
+  // All the implementation detail is in the base class.
+  /// Implement operator<< on DbgMarkerBase.
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+  LLVM_ABI void dump() const;
+};
+
+class DbgMachineMarker : public DbgMarkerBase<DbgMachineRecord, MachineInstr, MachineBasicBlock, DbgMachineMarker> {
+public:
+  DbgMachineMarker() : DbgMarkerBase() {}
+  // All the implementation detail is in the base class.
+  /// Implement operator<< on DbgMarkerBase.
+  LLVM_ABI void print(raw_ostream &O, bool IsForDebug = false) const;
+  LLVM_ABI void print(raw_ostream &ROS, ModuleSlotTracker &MST,
+                      bool IsForDebug) const;
+  LLVM_ABI void dump() const;
+};
+
+extern template class DbgMarkerBase<DbgRecord, Instruction, BasicBlock, DbgMarker>;
+extern template class DbgMarkerBase<DbgMachineRecord, MachineInstr, MachineBasicBlock, DbgMachineMarker>;
+
 inline raw_ostream &operator<<(raw_ostream &OS, const DbgMarker &Marker) {
   Marker.print(OS);
   return OS;
@@ -690,8 +901,25 @@ getDbgRecordRange(DbgMarker *DebugMarker) {
   return DebugMarker->getDbgRecordRange();
 }
 
+inline iterator_range<simple_ilist<DbgMachineRecord>::iterator>
+getDbgRecordRange(DbgMachineMarker *DebugMarker) {
+  if (!DebugMarker)
+    return DbgMachineMarker::getEmptyDbgRecordRange();
+  return DebugMarker->getDbgRecordRange();
+}
+
 DEFINE_ISA_CONVERSION_FUNCTIONS(DbgRecord, LLVMDbgRecordRef)
 
+template <typename InstT, typename BlockT, typename FuncT, typename MarkerT, typename CRTP>
+auto DbgRecordBase<InstT, BlockT, FuncT, MarkerT, CRTP>::getInstruction() -> InstT* {
+  return Marker->MarkedInstr;
+}
+
+template <typename InstT, typename BlockT, typename FuncT, typename MarkerT, typename CRTP>
+auto DbgRecordBase<InstT, BlockT, FuncT, MarkerT, CRTP>::getInstruction() const -> const InstT* {
+  return Marker->MarkedInstr;
+}
+
 } // namespace llvm
 
 #endif // LLVM_IR_DEBUGPROGRAMINSTRUCTION_H
diff --git a/llvm/lib/IR/ZOSLibcallNames.def b/llvm/include/llvm/IR/ZOSLibcallNames.def
similarity index 100%
rename from llvm/lib/IR/ZOSLibcallNames.def
rename to llvm/include/llvm/IR/ZOSLibcallNames.def
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index c3c5a0f5102d7..086a5d0a31657 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -1461,6 +1461,7 @@ MachineInstr *MachineBasicBlock::remove_instr(MachineInstr *MI) {
 
 MachineBasicBlock::instr_iterator
 MachineBasicBlock::insert(instr_iterator I, MachineInstr *MI) {
+  assert(!MI->DebugMarker);
   assert(!MI->isBundledWithPred() && !MI->isBundledWithSucc() &&
          "Cannot insert instruction with bundle flags");
   // Set the bundle flags when inserting inside a bundle.
@@ -1468,7 +1469,30 @@ MachineBasicBlock::insert(instr_iterator I, MachineInstr *MI) {
     MI->setFlag(MachineInstr::BundledPred);
     MI->setFlag(MachineInstr::BundledSucc);
   }
-  return Insts.insert(I, MI);
+
+  MachineBasicBlock::instr_iterator Inserted = Insts.insert(I, MI);
+
+  // We've inserted MI: if InsertAtHead is set then it comes before any
+  // DbgRecords attached to InsertPos. But if it's not set, then any
+  // DbgRecords should now come before MI.
+  bool InsertAtHead = I.getHeadBit();
+  if (!InsertAtHead) {
+    DbgMachineMarker *SrcMarker = getMarker(I);
+    if (SrcMarker && !SrcMarker->empty()) {
+      // See corresponding comment in Instruction::insertBefore, having an
+      // intermingling of PHIs and debug records is de-normal.
+      assert(!MI->isPHI() && "Inserting PHI after debug-records!");
+      MI->adoptDbgRecords(this, I, false);
+    }
+  }
+
+  // If we're inserting a terminator, check if we need to flush out
+  // TrailingDbgRecords. Inserting instructions at the end of an incomplete
+  // block is handled by the code block above.
+  if (MI->isTerminator())
+    flushTerminatorDbgRecords();
+
+  return Inserted;
 }
 
 /// This method unlinks 'this' from the containing function, and returns it, but
@@ -1808,3 +1832,78 @@ bool MachineBasicBlock::sizeWithoutDebugLargerThan(unsigned Limit) const {
 const MBBSectionID MBBSectionID::ColdSectionID(MBBSectionID::SectionType::Cold);
 const MBBSectionID
     MBBSectionID::ExceptionSectionID(MBBSectionID::SectionType::Exception);
+
+DbgMachineMarker *MachineBasicBlock::getNextMarker(MachineInstr *I) {
+  return getMarker(std::next(I->getIterator()));
+}
+
+DbgMachineMarker *MachineBasicBlock::getMarker(iterator It) {
+  if (It == end()) {
+    DbgMachineMarker *DM = getTrailingDbgRecords();
+    return DM;
+  }
+  return It->DebugMarker;
+}
+
+void MachineBasicBlock::setTrailingDbgRecords(DbgMachineMarker *foo) {
+  getParent()->setTrailingDbgRecords(this, foo);
+}
+
+DbgMachineMarker *MachineBasicBlock::getTrailingDbgRecords() {
+  return getParent()->getTrailingDbgRecords(this);
+}
+
+void MachineBasicBlock::deleteTrailingDbgRecords() {
+  getParent()->deleteTrailingDbgRecords(this);
+}
+
+void MachineBasicBlock::flushTerminatorDbgRecords() {
+// XXX reword for MIs
+  // If we erase the terminator in a block, any DbgRecords will sink and "fall
+  // off the end", existing after any terminator that gets inserted. With
+  // dbg.value intrinsics we would just insert the terminator at end() and
+  // the dbg.values would come before the terminator. With DbgRecords, we must
+  // do this manually.
+  // To get out of this unfortunate form, whenever we insert a terminator,
+  // check whether there's anything trailing at the end and move those
+  // DbgRecords in front of the terminator.
+
+  // If there's no terminator, there's nothing to do.
+  MachineInstr *Term = &*getFirstTerminator();
+  if (!Term)
+    return;
+
+  // Are there any dangling DbgRecords?
+  DbgMachineMarker *TrailingDbgRecords = getTrailingDbgRecords();
+  if (!TrailingDbgRecords)
+    return;
+
+  // Transfer DbgRecords from the trailing position onto the terminator.
+  createMarker(Term);
+  Term->DebugMarker->absorbDebugValues(*TrailingDbgRecords, false);
+  TrailingDbgRecords->eraseFromParent();
+  deleteTrailingDbgRecords();
+}
+
+DbgMachineMarker *MachineBasicBlock::createMarker(MachineInstr *I) {
+  if (I->DebugMarker)
+    return I->DebugMarker;
+  DbgMachineMarker *Marker = new DbgMachineMarker();
+  Marker->MarkedInstr = I;
+  I->DebugMarker = Marker;
+  return Marker;
+}
+
+DbgMachineMarker *MachineBasicBlock::createMarker(iterator It) {
+  if (It != end())
+    return createMarker(&*It);
+
+  DbgMachineMarker *DM = getTrailingDbgRecords();
+  if (DM)
+    return DM;
+
+  DM = new DbgMachineMarker();
+  setTrailingDbgRecords(DM);
+  return DM;
+}
+
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 607e87a38274b..75da8e78fd980 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -1210,6 +1210,40 @@ auto MachineFunction::salvageCopySSAImpl(MachineInstr &MI)
   return ApplySubregisters({NewNum, 0u});
 }
 
+std::optional<std::pair<unsigned, unsigned>> MachineFunction::finalizeDebugInstrRefRef(Register R, DenseMap<Register, DebugInstrOperandPair> &ArgDbgPHIs) {
+  auto *TII = getSubtarget().getInstrInfo();
+  // Some vregs can be deleted as redundant in the meantime. Mark those
+  // as DBG_VALUE $noreg. Additionally, some normal instructions are
+  // quickly deleted, leaving dangling references to vregs with no def.
+  if (R == 0 || !RegInfo->hasOneDef(R)) {
+    return std::nullopt;
+  }
+
+  assert(R.isVirtual());
+  MachineInstr &DefMI = *RegInfo->def_instr_begin(R);
+
+  // If we've found a copy-like instruction, follow it back to the
+  // instruction that defines the source value, see salvageCopySSA docs
+  // for why this is important.
+  if (DefMI.isCopyLike() || TII->isCopyInstr(DefMI)) {
+    auto Result = salvageCopySSA(DefMI, ArgDbgPHIs);
+    return std::make_pair(Result.first, Result.second);
+  } else {
+    // Otherwise, identify the operand number that the VReg refers to.
+    unsigned OperandIdx = 0;
+    for (const auto &DefMO : DefMI.operands()) {
+      if (DefMO.isReg() && DefMO.isDef() && DefMO.getReg() == R)
+        break;
+      ++OperandIdx;
+    }
+    assert(OperandIdx < DefMI.getNumOperands());
+
+    // Morph this instr ref to point at the given instruction and operand.
+    unsigned ID = DefMI.getDebugInstrNum();
+    return std::make_pair(ID, OperandIdx);
+  }
+}
+
 void MachineFunction::finalizeDebugInstrRefs() {
   auto *TII = getSubtarget().getInstrInfo();
 
@@ -1219,9 +1253,33 @@ void MachineFunction::finalizeDebugInstrRefs() {
     MI.setDebugValueUndef();
   };
 
+  auto MakeUndefDbgRec = [&](DbgMachineVariableRecord *DMI) {
+    DMI->Type = DbgMachineVariableRecord::MachineLocationType::Value;
+    DMI->Reg = 0;
+  };
+
   DenseMap<Register, DebugInstrOperandPair> ArgDbgPHIs;
   for (auto &MBB : *this) {
     for (auto &MI : MBB) {
+      for (DbgMachineRecord &DbgMRec : MI.getDbgRecordRange()) {
+        DbgMachineVariableRecord *VarRec = dyn_cast<DbgMachineVariableRecord>(&DbgMRec);
+        if (!VarRec)
+          continue;
+        if (!VarRec->isRef())
+          continue;
+        for (auto &P : VarRec->Refs) {
+          if (P.second != UINT_MAX)
+            continue;
+          // It's a vreg, try to resolve.
+          auto lol = finalizeDebugInstrRefRef(P.first, ArgDbgPHIs);
+          if (!lol) {
+            MakeUndefDbgRec(VarRec);
+            break;
+          }
+          P = *lol;
+        }
+      }
+
       if (!MI.isDebugRef())
         continue;
 
@@ -1233,37 +1291,13 @@ void MachineFunction::finalizeDebugInstrRefs() {
 
         Register Reg = MO.getReg();
 
-        // Some vregs can be deleted as redundant in the meantime. Mark those
-        // as DBG_VALUE $noreg. Additionally, some normal instructions are
-        // quickly deleted, leaving dangling references to vregs with no def.
-        if (Reg == 0 || !RegInfo->hasOneDef(Reg)) {
+        auto lol = finalizeDebugInstrRefRef(Reg, ArgDbgPHIs);
+        if (!lol) {
           IsValidRef = false;
           break;
         }
 
-        assert(Reg.isVirtual());
-        MachineInstr &DefMI = *RegInfo->def_instr_begin(Reg);
-
-        // If we've found a copy-like instruction, follow it back to the
-        // instruction that defines the source value, see salvageCopySSA docs
-        // for why this is important.
-        if (DefMI.isCopyLike() || TII->isCopyInstr(DefMI)) {
-          auto Result = salvageCopySSA(DefMI, ArgDbgPHIs);
-          MO.ChangeToDbgInstrRef(Result.first, Result.second);
-        } else {
-          // Otherwise, identify the operand number that the VReg refers to.
-          unsigned OperandIdx = 0;
-          for (const auto &DefMO : DefMI.operands()) {
-            if (DefMO.isReg() && DefMO.isDef() && DefMO.getReg() == Reg)
-              break;
-            ++OperandIdx;
-          }
-          assert(OperandIdx < DefMI.getNumOperands());
-
-          // Morph this instr ref to point at the given instruction and operand.
-          unsigned ID = DefMI.getDebugInstrNum();
-          MO.ChangeToDbgInstrRef(ID, OperandIdx);
-        }
+        MO.ChangeToDbgInstrRef(lol->first, lol->second);
       }
 
       if (!IsValidRef)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index da3665b3b6a0b..876bcfacc3b22 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -767,6 +767,7 @@ MachineInstr *MachineInstr::removeFromBundle() {
 
 void MachineInstr::eraseFromParent() {
   assert(getParent() && "Not embedded in a basic block!");
+  handleMarkerRemoval();
   getParent()->erase(this);
 }
 
@@ -2684,6 +2685,7 @@ MachineInstr::getFirst5RegLLTs() const {
 
 void MachineInstr::insert(mop_iterator InsertBefore,
                           ArrayRef<MachineOperand> Ops) {
+  assert(!DebugMarker);
   assert(InsertBefore != nullptr && "invalid iterator");
   assert(InsertBefore->getParent() == this &&
          "iterator points to operand of other inst");
@@ -2743,3 +2745,54 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
     return F.getRegMayBeFolded();
   return false;
 }
+
+void MachineInstr::adoptDbgRecords(MachineBasicBlock *MBB, MachineBasicBlock::instr_iterator It,
+                                  bool InsertAtHead) {
+  DbgMachineMarker *SrcMarker = MBB->getMarker(It);
+  auto ReleaseTrailingDbgRecords = [MBB, It, SrcMarker]() {
+    if (MBB->end() == It) {
+      SrcMarker->eraseFromParent();
+      MBB->deleteTrailingDbgRecords();
+    }
+  };
+
+  if (!SrcMarker || SrcMarker->StoredDbgRecords.empty()) {
+    ReleaseTrailingDbgRecords();
+    return;
+  }
+
+  // If we have DbgMarkers attached to this instruction, we have to honour the
+  // ordering of DbgRecords between this and the other marker. Fall back to just
+  // absorbing from the source.
+  if (DebugMarker || It == MBB->end()) {
+    // Ensure we _do_ have a marker.
+    getParent()->createMarker(this);
+    DebugMarker->absorbDebugValues(*SrcMarker, InsertAtHead);
+
+    // Having transferred everything out of SrcMarker, we _could_ clean it up
+    // and free the marker now. However, that's a lot of heap-accounting for a
+    // small amount of memory with a good chance of re-use. Leave it for the
+    // moment. It will be released when the Instruction is freed in the worst
+    // case.
+    // However: if we transferred from a trailing marker off the end of the
+    // block, it's important to not leave the empty marker trailing. It will
+    // give a misleading impression that some debug records have been left
+    // trailing.
+    ReleaseTrailingDbgRecords();
+  } else {
+    // Optimisation: we're transferring all the DbgRecords from the source
+    // marker onto this empty location: just adopt the other instructions
+    // marker.
+    DebugMarker = SrcMarker;
+    DebugMarker->MarkedInstr = this;
+    It->DebugMarker = nullptr;
+  }
+}
+
+void MachineInstr::handleMarkerRemoval() {
+  if (!DebugMarker)
+    return;
+
+  DebugMarker->removeMarker();
+}
+
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 4b7a9127b3fc3..3856ec1699aa1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -31,6 +31,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "instr-emitter"
 
+bool UsingDDDISel = true;
+
 /// MinRCSize - Smallest register class we allow when constraining virtual
 /// registers.  If satisfying all register class constraints would require
 /// using a smaller register class, emit a COPY to a new virtual register
@@ -696,7 +698,7 @@ void InstrEmitter::EmitRegSequence(SDNode *Node, VRBaseMapType &VRBaseMap,
 
 /// EmitDbgValue - Generate machine instruction for a dbg_value node.
 ///
-MachineInstr *
+std::variant<MachineInstr *, DbgMachineRecord *>
 InstrEmitter::EmitDbgValue(SDDbgValue *SD,
                            VRBaseMapType &VRBaseMap) {
   DebugLoc DL = SD->getDebugLoc();
@@ -713,9 +715,13 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD,
     return EmitDbgNoLocation(SD);
 
   // Attempt to produce a DBG_INSTR_REF if we've been asked to.
-  if (EmitDebugInstrRefs)
-    if (auto *InstrRef = EmitDbgInstrRef(SD, VRBaseMap))
+  if (EmitDebugInstrRefs) {
+    auto InstrRef = EmitDbgInstrRef(SD, VRBaseMap);
+    // If it's holding a debug-record, or a non-null MachineInstr *, emit that.
+    if (!std::holds_alternative<MachineInstr*>(InstrRef) ||
+        std::get<MachineInstr*>(InstrRef))
       return InstrRef;
+  }
 
   // Emit variadic dbg_value nodes as DBG_VALUE_LIST if they have not been
   // emitted as instruction references.
@@ -779,7 +785,7 @@ void InstrEmitter::AddDbgValueLocationOps(
   }
 }
 
-MachineInstr *
+std::variant<MachineInstr *, DbgMachineRecord *>
 InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
                               VRBaseMapType &VRBaseMap) {
   MDNode *Var = SD->getVariable();
@@ -817,6 +823,7 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
     Expr = DIExpression::convertToVariadicExpression(Expr);
 
   SmallVector<MachineOperand> MOs;
+  SmallVector<std::pair<unsigned, unsigned>> Refs;
 
   // It may not be immediately possible to identify the MachineInstr that
   // defines a VReg, it can depend for example on the order blocks are
@@ -827,13 +834,19 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
   //
   // i.e., point the instruction at the vreg, and patch it up later in
   // MachineFunction::finalizeDebugInstrRefs.
-  auto AddVRegOp = [&](Register VReg) {
-    MOs.push_back(MachineOperand::CreateReg(
-        /* Reg */ VReg, /* isDef */ false, /* isImp */ false,
-        /* isKill */ false, /* isDead */ false,
-        /* isUndef */ false, /* isEarlyClobber */ false,
-        /* SubReg */ 0, /* isDebug */ true));
+  auto RecordVRegOperand = [&](Register R) {
+    if (!UsingDDDISel) {
+      MOs.push_back(MachineOperand::CreateReg(
+          /* Reg */ R, /* isDef */ false, /* isImp */ false,
+          /* isKill */ false, /* isDead */ false,
+          /* isUndef */ false, /* isEarlyClobber */ false,
+          /* SubReg */ 0, /* isDebug */ true));
+    } else {
+      // Otherwise, uh, record some "special" values.
+      Refs.push_back(std::make_pair(R, UINT_MAX));
+    }
   };
+
   unsigned OpCount = SD->getLocationOps().size();
   for (unsigned OpIdx = 0; OpIdx < OpCount; ++OpIdx) {
     SDDbgOperand DbgOperand = SD->getLocationOps()[OpIdx];
@@ -848,7 +861,7 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
       // No definition means that block hasn't been emitted yet. Leave a vreg
       // reference to be fixed later.
       if (!MRI->hasOneDef(VReg)) {
-        AddVRegOp(VReg);
+        RecordVRegOperand(VReg);
         continue;
       }
 
@@ -868,12 +881,13 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
       // Again, if there's no instruction defining the VReg right now, fix it up
       // later.
       if (!MRI->hasOneDef(VReg)) {
-        AddVRegOp(VReg);
+        RecordVRegOperand(VReg);
         continue;
       }
 
       DefMI = &*MRI->def_instr_begin(VReg);
     } else {
+llvm_unreachable("I thought we were getting rid of these?");
       assert(DbgOperand.getKind() == SDDbgOperand::CONST);
       MOs.push_back(GetMOForConstDbgOp(DbgOperand));
       continue;
@@ -883,7 +897,7 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
     // Leave a virtual-register reference until it can be fixed up later, to
     // find the underlying value definition.
     if (DefMI->isCopyLike() || TII->isCopyInstr(*DefMI)) {
-      AddVRegOp(VReg);
+      RecordVRegOperand(VReg);
       continue;
     }
 
@@ -898,15 +912,26 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
 
     // Make the DBG_INSTR_REF refer to that instruction, and that operand.
     unsigned InstrNum = DefMI->getDebugInstrNum();
-    MOs.push_back(MachineOperand::CreateDbgInstrRef(InstrNum, OperandIdx));
+    if (!UsingDDDISel) {
+      MOs.push_back(MachineOperand::CreateDbgInstrRef(InstrNum, OperandIdx));
+    } else {
+      assert(OperandIdx != UINT_MAX);
+      Refs.push_back(std::make_pair(InstrNum, OperandIdx));
+    }
   }
 
   // If we haven't created a valid MachineOperand for every DbgOp, abort and
   // produce an undef DBG_VALUE.
-  if (MOs.size() != OpCount)
+  if (!UsingDDDISel && MOs.size() != OpCount)
+    return EmitDbgNoLocation(SD);
+  if (UsingDDDISel && Refs.size() != OpCount)
     return EmitDbgNoLocation(SD);
 
-  return BuildMI(*MF, DL, RefII, false, MOs, Var, Expr);
+  if (!UsingDDDISel) {
+    return BuildMI(*MF, DL, RefII, false, MOs, Var, Expr);
+  } else {
+    return DbgMachineVariableRecord::createDMVRRef(Refs, cast<DILocalVariable>(Var), (DIExpression*)Expr, DL);
+  }
 }
 
 MachineInstr *InstrEmitter::EmitDbgNoLocation(SDDbgValue *SD) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
index 16d754cdc2338..79857b3781cae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -15,6 +15,8 @@
 #ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_INSTREMITTER_H
 #define LLVM_LIB_CODEGEN_SELECTIONDAG_INSTREMITTER_H
 
+#include <variant>
+
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -117,12 +119,12 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
 
   /// EmitDbgValue - Generate machine instruction for a dbg_value node.
   ///
-  MachineInstr *EmitDbgValue(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
+  std::variant<MachineInstr *, DbgMachineRecord *> EmitDbgValue(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
 
   /// Emit a dbg_value as a DBG_INSTR_REF. May produce DBG_VALUE $noreg instead
   /// if there is no variable location; alternately a half-formed DBG_INSTR_REF
   /// that refers to a virtual register and is corrected later in isel.
-  MachineInstr *EmitDbgInstrRef(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
+  std::variant<MachineInstr *, DbgMachineRecord *> EmitDbgInstrRef(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
 
   /// Emit a DBG_VALUE $noreg, indicating a variable has no location.
   MachineInstr *EmitDbgNoLocation(SDDbgValue *SD);
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index 5af3df161b6c5..7687335dea650 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -778,9 +778,21 @@ ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
     if (N->getHasDebugValue()) {
       MachineBasicBlock::iterator InsertPos = Emitter.getInsertPos();
       for (auto *DV : DAG->GetDbgValues(N)) {
-        if (!DV->isEmitted())
-          if (auto *DbgMI = Emitter.EmitDbgValue(DV, VRBaseMap))
-            BB->insert(InsertPos, DbgMI);
+        if (!DV->isEmitted()) {
+          auto DbgMI = Emitter.EmitDbgValue(DV, VRBaseMap);
+          if (std::holds_alternative<MachineInstr*>(DbgMI)) {
+            MachineInstr *MI = std::get<MachineInstr*>(DbgMI);
+            if (!MI)
+              continue;
+            BB->insert(InsertPos, MI);
+          } else {
+            // It's a DDD record!
+            DbgMachineRecord *DMVR = std::get<DbgMachineRecord*>(DbgMI);
+            DbgMachineMarker *Marker = BB->createMarker(&*InsertPos);
+            // Insert this at the end.
+            Marker->insertDbgRecord(DMVR, false);
+          }
+        }
       }
     }
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 6a2e782fc688e..a4b97245d2379 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -37,6 +37,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "pre-RA-sched"
 
+extern bool UsingDDDISel;
+
 STATISTIC(LoadsClustered, "Number of loads clustered together");
 
 // This allows the latency-based scheduler to notice high latency instructions
@@ -768,11 +770,20 @@ ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
     // dependent nodes have been visited.
     if (!DV->isInvalidated() && HasUnknownVReg(DV))
       continue;
-    MachineInstr *DbgMI = Emitter.EmitDbgValue(DV, VRBaseMap);
-    if (!DbgMI)
-      continue;
-    Orders.push_back({DVOrder, DbgMI});
-    BB->insert(InsertPos, DbgMI);
+    auto DbgMI = Emitter.EmitDbgValue(DV, VRBaseMap);
+    if (std::holds_alternative<MachineInstr*>(DbgMI)) {
+      MachineInstr *MI = std::get<MachineInstr*>(DbgMI);
+      if (!MI)
+        continue;
+      Orders.push_back({DVOrder, MI});
+      BB->insert(InsertPos, MI);
+    } else {
+      // It's a DDD record!
+      DbgMachineRecord *DMVR = std::get<DbgMachineRecord*>(DbgMI);
+      DbgMachineMarker *Marker = BB->createMarker(InsertPos);
+      // Insert this at the end.
+      Marker->insertDbgRecord(DMVR, false);
+    }
   }
 }
 
@@ -919,12 +930,22 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
     SDDbgInfo::DbgIterator PDI = DAG->ByvalParmDbgBegin();
     SDDbgInfo::DbgIterator PDE = DAG->ByvalParmDbgEnd();
     for (; PDI != PDE; ++PDI) {
-      MachineInstr *DbgMI= Emitter.EmitDbgValue(*PDI, VRBaseMap);
-      if (DbgMI) {
-        BB->insert(InsertPos, DbgMI);
+      auto DbgMI= Emitter.EmitDbgValue(*PDI, VRBaseMap);
+      if (std::holds_alternative<MachineInstr*>(DbgMI)) {
+        MachineInstr *MI = std::get<MachineInstr*>(DbgMI);
+        if (!MI)
+          continue;
+        BB->insert(InsertPos, MI);
         // We re-emit the dbg_value closer to its use, too, after instructions
         // are emitted to the BB.
         (*PDI)->clearIsEmitted();
+      } else {
+        // It's a DDD record!
+        DbgMachineRecord *DMVR = std::get<DbgMachineRecord*>(DbgMI);
+        DbgMachineMarker *Marker = BB->createMarker(&*InsertPos);
+        // Insert this at the end.
+        Marker->insertDbgRecord(DMVR, false);
+        (*PDI)->clearIsEmitted();
       }
     }
   }
@@ -1002,36 +1023,68 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
         if ((*DI)->isEmitted())
           continue;
 
-        MachineInstr *DbgMI = Emitter.EmitDbgValue(*DI, VRBaseMap);
-        if (DbgMI) {
+        auto DbgMI = Emitter.EmitDbgValue(*DI, VRBaseMap);
+
+        if (std::holds_alternative<MachineInstr*>(DbgMI)) {
+          MachineInstr *NewDbgMI = std::get<MachineInstr*>(DbgMI);
+          if (!MI)
+            continue;
+
           if (!LastOrder)
             // Insert to start of the BB (after PHIs).
-            BB->insert(BBBegin, DbgMI);
+            BB->insert(BBBegin, NewDbgMI);
           else {
             // Insert at the instruction, which may be in a different
             // block, if the block was split by a custom inserter.
             MachineBasicBlock::iterator Pos = MI;
-            MI->getParent()->insert(Pos, DbgMI);
+            MI->getParent()->insert(Pos, NewDbgMI);
+          }
+        } else {
+          // It's a DDD record!
+          DbgMachineRecord *DMVR = std::get<DbgMachineRecord*>(DbgMI);
+          DbgMachineMarker *Marker;
+          if (!LastOrder)
+            // Insert to start of the BB (after PHIs).
+            Marker = BB->createMarker(BBBegin);
+          else {
+            // Insert at the instruction, which may be in a different
+            // block, if the block was split by a custom inserter.
+            MachineBasicBlock::iterator Pos = MI;
+            Marker = MI->getParent()->createMarker(&*Pos);
           }
+
+          // Insert this at the end.
+          Marker->insertDbgRecord(DMVR, false);
         }
       }
       LastOrder = Order;
     }
     // Add trailing DbgValue's before the terminator. FIXME: May want to add
     // some of them before one or more conditional branches?
-    SmallVector<MachineInstr*, 8> DbgMIs;
+    SmallVector<std::variant<MachineInstr *, DbgMachineRecord *>, 8> DbgMIs;
     for (; DI != DE; ++DI) {
       if ((*DI)->isEmitted())
         continue;
       assert((*DI)->getOrder() >= LastOrder &&
              "emitting DBG_VALUE out of order");
-      if (MachineInstr *DbgMI = Emitter.EmitDbgValue(*DI, VRBaseMap))
-        DbgMIs.push_back(DbgMI);
+      auto DbgMI = Emitter.EmitDbgValue(*DI, VRBaseMap);
+      DbgMIs.push_back(DbgMI);
     }
 
     MachineBasicBlock *InsertBB = Emitter.getBlock();
     MachineBasicBlock::iterator Pos = InsertBB->getFirstTerminator();
-    InsertBB->insert(Pos, DbgMIs.begin(), DbgMIs.end());
+    for (auto &Variant : llvm::reverse(DbgMIs)) { 
+      if (std::holds_alternative<MachineInstr *>(Variant)) {
+        MachineInstr *DbgMI = std::get<MachineInstr *>(Variant);
+        if (!DbgMI)
+          continue;
+        InsertBB->insert(Pos, DbgMI);
+      } else {
+        DbgMachineMarker *DbgMarker = InsertBB->createMarker(Pos);
+        DbgMachineRecord *DbgMI = std::get<DbgMachineRecord *>(Variant);
+        DbgMarker->insertDbgRecord(DbgMI, false);
+      }
+    }
 
     SDDbgInfo::DbgLabelIterator DLI = DAG->DbgLabelBegin();
     SDDbgInfo::DbgLabelIterator DLE = DAG->DbgLabelEnd();
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 196fe294a274b..6dd32a2dd5ee5 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -194,7 +194,7 @@ DebugLoc llvm::getDebugValueLoc(DbgVariableRecord *DVR) {
   // and inlinedAt is significant. Zero line numbers are used in case this
   // DebugLoc leaks into any adjacent instructions. Produce an unknown location
   // with the correct scope / inlinedAt fields.
-  return DILocation::get(DVR->getContext(), 0, 0, Scope, InlinedAt);
+  return DILocation::get(DeclareLoc->getContext(), 0, 0, Scope, InlinedAt);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp
index 2b9b0f958a171..e4248f9e64bc2 100644
--- a/llvm/lib/IR/DebugProgramInstruction.cpp
+++ b/llvm/lib/IR/DebugProgramInstruction.cpp
@@ -12,6 +12,8 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Support/Compiler.h"
 
+#include "llvm/CodeGen/MachineBasicBlock.h"
+
 namespace llvm {
 
 template <typename T>
@@ -128,10 +130,6 @@ bool DbgRecord::isIdenticalToWhenDefined(const DbgRecord &R) const {
   llvm_unreachable("unsupported DbgRecord kind");
 }
 
-bool DbgRecord::isEquivalentTo(const DbgRecord &R) const {
-  return getDebugLoc() == R.getDebugLoc() && isIdenticalToWhenDefined(R);
-}
-
 DbgInfoIntrinsic *
 DbgRecord::createDebugIntrinsic(Module *M, Instruction *InsertBefore) const {
   switch (RecordKind) {
@@ -501,128 +499,73 @@ bool DbgVariableRecord::isKillAddress() const {
   return !Addr || isa<UndefValue>(Addr);
 }
 
-const Instruction *DbgRecord::getInstruction() const {
-  return Marker->MarkedInstr;
-}
-
-const BasicBlock *DbgRecord::getParent() const {
-  return Marker->MarkedInstr->getParent();
-}
-
-BasicBlock *DbgRecord::getParent() { return Marker->MarkedInstr->getParent(); }
 
-BasicBlock *DbgRecord::getBlock() { return Marker->getParent(); }
-
-const BasicBlock *DbgRecord::getBlock() const { return Marker->getParent(); }
-
-Function *DbgRecord::getFunction() { return getBlock()->getParent(); }
-
-const Function *DbgRecord::getFunction() const {
-  return getBlock()->getParent();
-}
-
-Module *DbgRecord::getModule() { return getFunction()->getParent(); }
-
-const Module *DbgRecord::getModule() const {
-  return getFunction()->getParent();
-}
-
-LLVMContext &DbgRecord::getContext() { return getBlock()->getContext(); }
-
-const LLVMContext &DbgRecord::getContext() const {
-  return getBlock()->getContext();
+void DbgMachineRecord::deleteRecord() {
+abort();
+#if 0
+  switch (RecordKind) {
+  case ValueKind:
+    delete cast<DbgVariableRecord>(this);
+    return;
+  case LabelKind:
+    delete cast<DbgLabelRecord>(this);
+    return;
+  }
+  llvm_unreachable("unsupported DbgRecord kind");
+#endif
 }
 
-void DbgRecord::insertBefore(DbgRecord *InsertBefore) {
-  assert(!getMarker() &&
-         "Cannot insert a DbgRecord that is already has a DbgMarker!");
-  assert(InsertBefore->getMarker() &&
-         "Cannot insert a DbgRecord before a DbgRecord that does not have a "
-         "DbgMarker!");
-  InsertBefore->getMarker()->insertDbgRecord(this, InsertBefore);
-}
-void DbgRecord::insertAfter(DbgRecord *InsertAfter) {
-  assert(!getMarker() &&
-         "Cannot insert a DbgRecord that is already has a DbgMarker!");
-  assert(InsertAfter->getMarker() &&
-         "Cannot insert a DbgRecord after a DbgRecord that does not have a "
-         "DbgMarker!");
-  InsertAfter->getMarker()->insertDbgRecordAfter(this, InsertAfter);
+DbgMachineRecord *DbgMachineRecord::clone() const {
+abort();
+#if 0
+  switch (RecordKind) {
+  case ValueKind:
+    return cast<DbgVariableRecord>(this)->clone();
+  case LabelKind:
+    return cast<DbgLabelRecord>(this)->clone();
+  };
+  llvm_unreachable("unsupported DbgRecord kind");
+#endif
 }
 
-void DbgRecord::insertBefore(self_iterator InsertBefore) {
-  assert(!getMarker() &&
-         "Cannot insert a DbgRecord that is already has a DbgMarker!");
-  assert(InsertBefore->getMarker() &&
-         "Cannot insert a DbgRecord before a DbgRecord that does not have a "
-         "DbgMarker!");
-  InsertBefore->getMarker()->insertDbgRecord(this, &*InsertBefore);
-}
-void DbgRecord::insertAfter(self_iterator InsertAfter) {
-  assert(!getMarker() &&
-         "Cannot insert a DbgRecord that is already has a DbgMarker!");
-  assert(InsertAfter->getMarker() &&
-         "Cannot insert a DbgRecord after a DbgRecord that does not have a "
-         "DbgMarker!");
-  InsertAfter->getMarker()->insertDbgRecordAfter(this, &*InsertAfter);
-}
 
-void DbgRecord::moveBefore(DbgRecord *MoveBefore) {
-  assert(getMarker() &&
-         "Canot move a DbgRecord that does not currently have a DbgMarker!");
-  removeFromParent();
-  insertBefore(MoveBefore);
-}
-void DbgRecord::moveAfter(DbgRecord *MoveAfter) {
-  assert(getMarker() &&
-         "Canot move a DbgRecord that does not currently have a DbgMarker!");
-  removeFromParent();
-  insertAfter(MoveAfter);
-}
 
-void DbgRecord::moveBefore(self_iterator MoveBefore) {
-  assert(getMarker() &&
-         "Canot move a DbgRecord that does not currently have a DbgMarker!");
-  removeFromParent();
-  insertBefore(MoveBefore);
-}
-void DbgRecord::moveAfter(self_iterator MoveAfter) {
-  assert(getMarker() &&
-         "Canot move a DbgRecord that does not currently have a DbgMarker!");
-  removeFromParent();
-  insertAfter(MoveAfter);
-}
 
 ///////////////////////////////////////////////////////////////////////////////
 
 // An empty, global, DbgMarker for the purpose of describing empty ranges of
 // DbgRecords.
-DbgMarker DbgMarker::EmptyDbgMarker;
+//DbgMarker DbgMarker::EmptyDbgMarker;
 
-void DbgMarker::dropDbgRecords() {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::dropDbgRecords() {
   while (!StoredDbgRecords.empty()) {
     auto It = StoredDbgRecords.begin();
-    DbgRecord *DR = &*It;
+    RecT *DR = &*It;
     StoredDbgRecords.erase(It);
     DR->deleteRecord();
   }
 }
 
-void DbgMarker::dropOneDbgRecord(DbgRecord *DR) {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::dropOneDbgRecord(RecT *DR) {
   assert(DR->getMarker() == this);
   StoredDbgRecords.erase(DR->getIterator());
   DR->deleteRecord();
 }
 
-const BasicBlock *DbgMarker::getParent() const {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+const BlockT *DbgMarkerBase<RecT, InstT, BlockT, CRTP>::getParent() const {
   return MarkedInstr->getParent();
 }
 
-BasicBlock *DbgMarker::getParent() { return MarkedInstr->getParent(); }
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+BlockT *DbgMarkerBase<RecT, InstT, BlockT, CRTP>::getParent() { return MarkedInstr->getParent(); }
 
-void DbgMarker::removeMarker() {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::removeMarker() {
   // Are there any DbgRecords in this DbgMarker? If not, nothing to preserve.
-  Instruction *Owner = MarkedInstr;
+  InstT *Owner = MarkedInstr;
   if (StoredDbgRecords.empty()) {
     eraseFromParent();
     Owner->DebugMarker = nullptr;
@@ -632,7 +575,7 @@ void DbgMarker::removeMarker() {
   // The attached DbgRecords need to be preserved; attach them to the next
   // instruction. If there isn't a next instruction, put them on the
   // "trailing" list.
-  DbgMarker *NextMarker = Owner->getParent()->getNextMarker(Owner);
+  DbgMarkerBase *NextMarker = Owner->getParent()->getNextMarker(Owner);
   if (NextMarker) {
     NextMarker->absorbDebugValues(*this, true);
     eraseFromParent();
@@ -640,79 +583,80 @@ void DbgMarker::removeMarker() {
     // We can avoid a deallocation -- just store this marker onto the next
     // instruction. Unless we're at the end of the block, in which case this
     // marker becomes the trailing marker of a degenerate block.
-    BasicBlock::iterator NextIt = std::next(Owner->getIterator());
+    typename BlockT::iterator NextIt = std::next(Owner->getIterator());
+    CRTP *Derived = static_cast<CRTP*>(this);
     if (NextIt == getParent()->end()) {
-      getParent()->setTrailingDbgRecords(this);
+      getParent()->setTrailingDbgRecords(Derived);
       MarkedInstr = nullptr;
     } else {
-      NextIt->DebugMarker = this;
+      NextIt->DebugMarker = Derived;
       MarkedInstr = &*NextIt;
     }
   }
   Owner->DebugMarker = nullptr;
 }
 
-void DbgMarker::removeFromParent() {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::removeFromParent() {
   MarkedInstr->DebugMarker = nullptr;
   MarkedInstr = nullptr;
 }
 
-void DbgMarker::eraseFromParent() {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::eraseFromParent() {
   if (MarkedInstr)
     removeFromParent();
   dropDbgRecords();
   delete this;
 }
 
-iterator_range<DbgRecord::self_iterator> DbgMarker::getDbgRecordRange() {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+iterator_range<typename simple_ilist<RecT>::iterator>
+DbgMarkerBase<RecT, InstT, BlockT, CRTP>::getDbgRecordRange() {
   return make_range(StoredDbgRecords.begin(), StoredDbgRecords.end());
 }
-iterator_range<DbgRecord::const_self_iterator>
-DbgMarker::getDbgRecordRange() const {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+iterator_range<typename simple_ilist<RecT>::const_iterator>
+DbgMarkerBase<RecT, InstT, BlockT, CRTP>::getDbgRecordRange() const {
   return make_range(StoredDbgRecords.begin(), StoredDbgRecords.end());
 }
 
-void DbgRecord::removeFromParent() {
-  getMarker()->StoredDbgRecords.erase(getIterator());
-  Marker = nullptr;
-}
-
-void DbgRecord::eraseFromParent() {
-  removeFromParent();
-  deleteRecord();
-}
-
-void DbgMarker::insertDbgRecord(DbgRecord *New, bool InsertAtHead) {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::insertDbgRecord(RecT *New, bool InsertAtHead) {
   auto It = InsertAtHead ? StoredDbgRecords.begin() : StoredDbgRecords.end();
   StoredDbgRecords.insert(It, *New);
-  New->setMarker(this);
+  New->setMarker(static_cast<CRTP*>(this));
 }
-void DbgMarker::insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore) {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::insertDbgRecord(RecT *New, RecT *InsertBefore) {
   assert(InsertBefore->getMarker() == this &&
          "DbgRecord 'InsertBefore' must be contained in this DbgMarker!");
   StoredDbgRecords.insert(InsertBefore->getIterator(), *New);
-  New->setMarker(this);
+  New->setMarker(static_cast<CRTP*>(this));
 }
-void DbgMarker::insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter) {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::insertDbgRecordAfter(RecT *New, RecT *InsertAfter) {
   assert(InsertAfter->getMarker() == this &&
          "DbgRecord 'InsertAfter' must be contained in this DbgMarker!");
   StoredDbgRecords.insert(++(InsertAfter->getIterator()), *New);
-  New->setMarker(this);
+  New->setMarker(static_cast<CRTP*>(this));
 }
 
-void DbgMarker::absorbDebugValues(DbgMarker &Src, bool InsertAtHead) {
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::absorbDebugValues(DbgMarkerBase &Src, bool InsertAtHead) {
   auto It = InsertAtHead ? StoredDbgRecords.begin() : StoredDbgRecords.end();
-  for (DbgRecord &DVR : Src.StoredDbgRecords)
-    DVR.setMarker(this);
+  for (RecT &DVR : Src.StoredDbgRecords)
+    DVR.setMarker(static_cast<CRTP*>(this));
 
   StoredDbgRecords.splice(It, Src.StoredDbgRecords);
 }
 
-void DbgMarker::absorbDebugValues(
-    iterator_range<DbgRecord::self_iterator> Range, DbgMarker &Src,
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+void DbgMarkerBase<RecT, InstT, BlockT, CRTP>::absorbDebugValues(
+    iterator_range<typename RecT::self_iterator> Range, DbgMarkerBase &Src,
     bool InsertAtHead) {
-  for (DbgRecord &DR : Range)
-    DR.setMarker(this);
+  for (RecT &DR : Range)
+    DR.setMarker(static_cast<CRTP*>(this));
 
   auto InsertPos =
       (InsertAtHead) ? StoredDbgRecords.begin() : StoredDbgRecords.end();
@@ -721,10 +665,12 @@ void DbgMarker::absorbDebugValues(
                           Range.end());
 }
 
-iterator_range<simple_ilist<DbgRecord>::iterator> DbgMarker::cloneDebugInfoFrom(
-    DbgMarker *From, std::optional<simple_ilist<DbgRecord>::iterator> from_here,
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+iterator_range<typename simple_ilist<RecT>::iterator>
+DbgMarkerBase<RecT, InstT, BlockT, CRTP>::cloneDebugInfoFrom(
+    DbgMarkerBase *From, std::optional<typename simple_ilist<RecT>::iterator> from_here,
     bool InsertAtHead) {
-  DbgRecord *First = nullptr;
+  RecT *First = nullptr;
   // Work out what range of DbgRecords to clone: normally all the contents of
   // the "From" marker, optionally we can start from the from_here position down
   // to end().
@@ -736,9 +682,9 @@ iterator_range<simple_ilist<DbgRecord>::iterator> DbgMarker::cloneDebugInfoFrom(
   // Clone each DbgVariableRecord and insert into StoreDbgVariableRecords;
   // optionally place them at the start or the end of the list.
   auto Pos = (InsertAtHead) ? StoredDbgRecords.begin() : StoredDbgRecords.end();
-  for (DbgRecord &DR : Range) {
-    DbgRecord *New = DR.clone();
-    New->setMarker(this);
+  for (RecT &DR : Range) {
+    RecT *New = DR.clone();
+    New->setMarker(static_cast<CRTP*>(this));
     StoredDbgRecords.insert(Pos, *New);
     if (!First)
       First = New;
@@ -756,4 +702,43 @@ iterator_range<simple_ilist<DbgRecord>::iterator> DbgMarker::cloneDebugInfoFrom(
     return {First->getIterator(), StoredDbgRecords.end()};
 }
 
+
+template <typename RecT, typename InstT, typename BlockT, typename CRTP>
+DbgMarkerBase<RecT, InstT, BlockT, CRTP> DbgMarkerBase<RecT, InstT, BlockT, CRTP>::EmptyDbgMarker = {};
+
+template class DbgMarkerBase<DbgRecord, Instruction, BasicBlock, DbgMarker>;
+template class DbgMarkerBase<DbgMachineRecord, MachineInstr, MachineBasicBlock, DbgMachineMarker>;
+
+DbgMachineVariableRecord::DbgMachineVariableRecord(DILocalVariable *Var, DIExpression *Expr, const DILocation *DILoc)
+  : DbgMachineRecord(ValueKind, DILoc), Variable(Var), Expression(Expr) {
+  Reg = Register(0);
+}
+
+DbgMachineVariableRecord *
+DbgMachineVariableRecord::createDMVRValue(Register R, DILocalVariable *Variable,
+                  DIExpression *Expression, const DILocation *DI) {
+  auto *NewThing = new DbgMachineVariableRecord(Variable, Expression, DI);
+  NewThing->Reg = R;
+  NewThing->Type = MachineLocationType::Value;
+  return NewThing;
+}
+
+DbgMachineVariableRecord *
+DbgMachineVariableRecord::createDMVRRef(ArrayRef<std::pair<unsigned, unsigned>> Refs,
+           DILocalVariable *DV, DIExpression *Expr, const DILocation *DI) {
+  auto *NewThing = new DbgMachineVariableRecord(DV, Expr, DI);
+  NewThing->Refs.insert(NewThing->Refs.begin(), Refs.begin(), Refs.end());
+  NewThing->Type = MachineLocationType::Ref;
+  return NewThing;
+}
+
+DbgMachineVariableRecord *
+DbgMachineVariableRecord::createDMVRPHI(Register R, DILocalVariable *Variable,
+           DIExpression *Expression, const DILocation *DI) {
+  auto *NewThing = new DbgMachineVariableRecord(Variable, Expression, DI);
+  NewThing->Reg = R;
+  NewThing->Type = MachineLocationType::PHI;
+  return NewThing;
+}
+
 } // end namespace llvm
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 7396626a03d41..7e810b2857985 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -609,7 +609,7 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
     };
     static RTLibCallMapping RTLibCallCommon[] = {
 #define HANDLE_LIBCALL(code, name) {RTLIB::code, name},
-#include "ZOSLibcallNames.def"
+#include "llvm/IR/ZOSLibcallNames.def"
     };
     for (auto &E : RTLibCallCommon)
       setLibcallName(E.Code, E.Name);
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 4a2eb9284a6ea..3e8333bb78501 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -542,16 +542,12 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest,
     return LinkToNothing;
   };
 
-  // Insert an unlinked dbg.assign intrinsic for the dead fragment after each
-  // overlapping dbg.assign intrinsic. The loop invalidates the iterators
-  // returned by getAssignmentMarkers so save a copy of the markers to iterate
-  // over.
-  auto LinkedRange = at::getAssignmentMarkers(Inst);
+  // Insert an unlinked DVRAssign for the dead fragment after each overlapping
+  // DVRAssign. The loop invalidates the iterators returned by
+  // getAssignmentMarkers so save a copy of the markers to iterate over.
   SmallVector<DbgVariableRecord *> LinkedDVRAssigns =
       at::getDVRAssignmentMarkers(Inst);
-  SmallVector<DbgAssignIntrinsic *> Linked(LinkedRange.begin(),
-                                           LinkedRange.end());
-  auto InsertAssignForOverlap = [&](auto *Assign) {
+  auto InsertAssignForOverlap = [&](DbgVariableRecord *Assign) {
     std::optional<DIExpression::FragmentInfo> NewFragment;
     if (!at::calculateFragmentIntersect(DL, OriginalDest, DeadSliceOffsetInBits,
                                         DeadSliceSizeInBits, Assign,
@@ -569,13 +565,12 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest,
 
     // Fragments overlap: insert a new dbg.assign for this dead part.
     auto *NewAssign = static_cast<decltype(Assign)>(Assign->clone());
-    NewAssign->insertAfter(Assign->getIterator());
+    NewAssign->insertAfter(Assign);
     NewAssign->setAssignId(GetDeadLink());
     if (NewFragment)
       SetDeadFragExpr(NewAssign, *NewFragment);
     NewAssign->setKillAddress();
   };
-  for_each(Linked, InsertAssignForOverlap);
   for_each(LinkedDVRAssigns, InsertAssignForOverlap);
 }
 
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 42d1d9a437bb2..c217781714c07 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -480,7 +480,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
     // noted as slightly offset (in code) from the store. In practice this
     // should have little effect on the debugging experience due to the fact
     // that all the split stores should get the same line number.
-    NewAssign->moveBefore(DbgAssign->getIterator());
+    NewAssign->moveBefore(DbgAssign);
 
     NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
     LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
diff --git a/llvm/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp b/llvm/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
index b583856fb559a..50be4915b68c0 100644
--- a/llvm/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
+++ b/llvm/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
@@ -15,7 +15,7 @@ using namespace llvm;
 namespace {
 
 struct MyBundledInstr
-    : public ilist_node<MyBundledInstr, ilist_sentinel_tracking<true>> {
+    : public ilist_node<MyBundledInstr, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>> {
   bool isBundledWithPred() const { return true; }
   bool isBundledWithSucc() const { return true; }
 };
@@ -130,7 +130,7 @@ TEST(MachineInstrBundleIteratorTest, CompareToBundledMI) {
 }
 
 struct MyUnbundledInstr
-    : ilist_node<MyUnbundledInstr, ilist_sentinel_tracking<true>> {
+    : ilist_node<MyUnbundledInstr, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>> {
   bool isBundledWithPred() const { return false; }
   bool isBundledWithSucc() const { return false; }
 };
@@ -143,7 +143,7 @@ typedef MachineInstrBundleIterator<const MyUnbundledInstr, true>
     const_reverse_unbundled_iterator;
 
 TEST(MachineInstrBundleIteratorTest, ReverseConstructor) {
-  simple_ilist<MyUnbundledInstr, ilist_sentinel_tracking<true>> L;
+  simple_ilist<MyUnbundledInstr, ilist_iterator_bits<true>, ilist_sentinel_tracking<true>> L;
   const auto &CL = L;
   MyUnbundledInstr A, B;
   L.insert(L.end(), A);