diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 7451ed6035471..3d430c0fc58e3 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -132,6 +132,7 @@ class LLVM_ABI MCSection { public: friend MCAssembler; friend MCObjectStreamer; + friend class MCEncodedFragment; static constexpr unsigned NonUniqueID = ~0U; enum SectionVariant { @@ -209,6 +210,10 @@ class LLVM_ABI MCSection { // subsections. SmallVector, 1> Subsections; + // Content and fixup storage for fragments + SmallVector ContentStorage; + SmallVector FixupStorage; + protected: // TODO Make Name private when possible. StringRef Name; @@ -296,9 +301,12 @@ class LLVM_ABI MCSection { /// Interface implemented by fragments that contain encoded instructions and/or /// data. -/// class MCEncodedFragment : public MCFragment { uint8_t BundlePadding = 0; + uint32_t ContentStart = 0; + uint32_t ContentEnd = 0; + uint32_t FixupStart = 0; + uint32_t FixupEnd = 0; protected: MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions) @@ -318,7 +326,10 @@ class MCEncodedFragment : public MCFragment { case MCFragment::FT_Data: case MCFragment::FT_Dwarf: case MCFragment::FT_DwarfFrame: + case MCFragment::FT_LEB: case MCFragment::FT_PseudoProbe: + case MCFragment::FT_CVInlineLines: + case MCFragment::FT_CVDefRange: return true; } } @@ -348,48 +359,64 @@ class MCEncodedFragment : public MCFragment { HasInstructions = true; this->STI = &STI; } -}; - -/// Interface implemented by fragments that contain encoded instructions and/or -/// data and also have fixups registered. -/// -template -class MCEncodedFragmentWithFixups : public MCEncodedFragment { - SmallVector Contents; - - /// The list of fixups in this fragment. - SmallVector Fixups; -protected: - MCEncodedFragmentWithFixups(MCFragment::FragmentType FType, - bool HasInstructions) - : MCEncodedFragment(FType, HasInstructions) {} - -public: - SmallVectorImpl &getContents() { return Contents; } - const SmallVectorImpl &getContents() const { return Contents; } - - void appendContents(ArrayRef C) { Contents.append(C.begin(), C.end()); } - void appendContents(size_t Num, char Elt) { Contents.append(Num, Elt); } - void setContents(ArrayRef C) { Contents.assign(C.begin(), C.end()); } - - void addFixup(MCFixup Fixup) { Fixups.push_back(Fixup); } - SmallVectorImpl &getFixups() { return Fixups; } - const SmallVectorImpl &getFixups() const { return Fixups; } + // Content-related functions manage parent's storage using ContentStart and + // ContentSize. + void clearContents() { ContentEnd = ContentStart; } + // Get a SmallVector reference. The caller should call doneAppending to update + // `ContentEnd`. + SmallVectorImpl &getContentsForAppending() { + SmallVectorImpl &S = getParent()->ContentStorage; + if (LLVM_UNLIKELY(ContentEnd != S.size())) { + // Move the elements to the end. Reserve space to avoid invalidating + // S.begin()+I for `append`. + auto Size = ContentEnd - ContentStart; + auto I = std::exchange(ContentStart, S.size()); + S.reserve(S.size() + Size); + S.append(S.begin() + I, S.begin() + I + Size); + } + return S; + } + void doneAppending() { ContentEnd = getParent()->ContentStorage.size(); } + void appendContents(ArrayRef Contents) { + getContentsForAppending().append(Contents.begin(), Contents.end()); + doneAppending(); + } + void appendContents(size_t Num, char Elt) { + getContentsForAppending().append(Num, Elt); + doneAppending(); + } + void setContents(ArrayRef Contents); + MutableArrayRef getContents() { + return MutableArrayRef(getParent()->ContentStorage) + .slice(ContentStart, ContentEnd - ContentStart); + } + ArrayRef getContents() const { + return ArrayRef(getParent()->ContentStorage) + .slice(ContentStart, ContentEnd - ContentStart); + } - static bool classof(const MCFragment *F) { - MCFragment::FragmentType Kind = F->getKind(); - return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data || - Kind == MCFragment::FT_CVDefRange || Kind == MCFragment::FT_Dwarf || - Kind == MCFragment::FT_DwarfFrame; + // Fixup-related functions manage parent's storage using FixupStart and + // FixupSize. + void clearFixups() { FixupEnd = FixupStart; } + void addFixup(MCFixup Fixup); + void appendFixups(ArrayRef Fixups); + void setFixups(ArrayRef Fixups); + MutableArrayRef getFixups() { + return MutableArrayRef(getParent()->FixupStorage) + .slice(FixupStart, FixupEnd - FixupStart); + } + ArrayRef getFixups() const { + return ArrayRef(getParent()->FixupStorage) + .slice(FixupStart, FixupEnd - FixupStart); } }; /// Fragment for data and encoded instructions. /// -class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> { +class MCDataFragment : public MCEncodedFragment { public: - MCDataFragment() : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false) {} + MCDataFragment() : MCEncodedFragment(FT_Data, false) {} static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_Data; @@ -402,13 +429,13 @@ class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> { /// A relaxable fragment holds on to its MCInst, since it may need to be /// relaxed during the assembler layout and relaxation stage. /// -class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> { +class MCRelaxableFragment : public MCEncodedFragment { /// The instruction this is a fragment for. MCInst Inst; public: MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI) - : MCEncodedFragmentWithFixups(FT_Relaxable, true), Inst(Inst) { + : MCEncodedFragment(FT_Relaxable, true), Inst(Inst) { this->STI = &STI; } @@ -557,7 +584,7 @@ class MCOrgFragment : public MCFragment { } }; -class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> { +class MCLEBFragment final : public MCEncodedFragment { /// True if this is a sleb128, false if uleb128. bool IsSigned; @@ -566,24 +593,19 @@ class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> { public: MCLEBFragment(const MCExpr &Value, bool IsSigned) - : MCEncodedFragmentWithFixups<8, 0>(FT_LEB, false), IsSigned(IsSigned), - Value(&Value) { - getContents().push_back(0); - } + : MCEncodedFragment(FT_LEB, false), IsSigned(IsSigned), Value(&Value) {} const MCExpr &getValue() const { return *Value; } void setValue(const MCExpr *Expr) { Value = Expr; } bool isSigned() const { return IsSigned; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_LEB; } }; -class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> { +class MCDwarfLineAddrFragment : public MCEncodedFragment { /// The value of the difference between the two line numbers /// between two .loc dwarf directives. int64_t LineDelta; @@ -594,8 +616,8 @@ class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> { public: MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta) - : MCEncodedFragmentWithFixups<8, 1>(FT_Dwarf, false), - LineDelta(LineDelta), AddrDelta(&AddrDelta) {} + : MCEncodedFragment(FT_Dwarf, false), LineDelta(LineDelta), + AddrDelta(&AddrDelta) {} int64_t getLineDelta() const { return LineDelta; } @@ -606,15 +628,14 @@ class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> { } }; -class MCDwarfCallFrameFragment : public MCEncodedFragmentWithFixups<8, 1> { +class MCDwarfCallFrameFragment : public MCEncodedFragment { /// The expression for the difference of the two symbols that /// make up the address delta between two .cfi_* dwarf directives. const MCExpr *AddrDelta; public: MCDwarfCallFrameFragment(const MCExpr &AddrDelta) - : MCEncodedFragmentWithFixups<8, 1>(FT_DwarfFrame, false), - AddrDelta(&AddrDelta) {} + : MCEncodedFragment(FT_DwarfFrame, false), AddrDelta(&AddrDelta) {} const MCExpr &getAddrDelta() const { return *AddrDelta; } void setAddrDelta(const MCExpr *E) { AddrDelta = E; } @@ -642,13 +663,12 @@ class MCSymbolIdFragment : public MCFragment { /// Fragment representing the binary annotations produced by the /// .cv_inline_linetable directive. -class MCCVInlineLineTableFragment : public MCFragment { +class MCCVInlineLineTableFragment : public MCEncodedFragment { unsigned SiteFuncId; unsigned StartFileId; unsigned StartLineNum; const MCSymbol *FnStartSym; const MCSymbol *FnEndSym; - SmallString<8> Contents; /// CodeViewContext has the real knowledge about this format, so let it access /// our members. @@ -658,23 +678,20 @@ class MCCVInlineLineTableFragment : public MCFragment { MCCVInlineLineTableFragment(unsigned SiteFuncId, unsigned StartFileId, unsigned StartLineNum, const MCSymbol *FnStartSym, const MCSymbol *FnEndSym) - : MCFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId), + : MCEncodedFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId), StartFileId(StartFileId), StartLineNum(StartLineNum), FnStartSym(FnStartSym), FnEndSym(FnEndSym) {} const MCSymbol *getFnStartSym() const { return FnStartSym; } const MCSymbol *getFnEndSym() const { return FnEndSym; } - SmallString<8> &getContents() { return Contents; } - const SmallString<8> &getContents() const { return Contents; } - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_CVInlineLines; } }; /// Fragment representing the .cv_def_range directive. -class MCCVDefRangeFragment : public MCEncodedFragmentWithFixups<32, 4> { +class MCCVDefRangeFragment : public MCEncodedFragment { ArrayRef> Ranges; StringRef FixedSizePortion; @@ -686,8 +703,9 @@ class MCCVDefRangeFragment : public MCEncodedFragmentWithFixups<32, 4> { MCCVDefRangeFragment( ArrayRef> Ranges, StringRef FixedSizePortion) - : MCEncodedFragmentWithFixups<32, 4>(FT_CVDefRange, false), - Ranges(Ranges), FixedSizePortion(FixedSizePortion) {} + : MCEncodedFragment(FT_CVDefRange, false), + Ranges(Ranges.begin(), Ranges.end()), + FixedSizePortion(FixedSizePortion) {} ArrayRef> getRanges() const { return Ranges; @@ -739,15 +757,14 @@ class MCBoundaryAlignFragment : public MCFragment { } }; -class MCPseudoProbeAddrFragment : public MCEncodedFragmentWithFixups<8, 1> { +class MCPseudoProbeAddrFragment : public MCEncodedFragment { /// The expression for the difference of the two symbols that /// make up the address delta between two .pseudoprobe directives. const MCExpr *AddrDelta; public: MCPseudoProbeAddrFragment(const MCExpr *AddrDelta) - : MCEncodedFragmentWithFixups<8, 1>(FT_PseudoProbe, false), - AddrDelta(AddrDelta) {} + : MCEncodedFragment(FT_PseudoProbe, false), AddrDelta(AddrDelta) {} const MCExpr &getAddrDelta() const { return *AddrDelta; } diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp index 995a68e5548c8..a9155935968b3 100644 --- a/llvm/lib/MC/MCAssembler.cpp +++ b/llvm/lib/MC/MCAssembler.cpp @@ -606,12 +606,14 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm, case MCFragment::FT_Data: ++stats::EmittedDataFragments; - OS << cast(F).getContents(); + OS << StringRef(cast(F).getContents().data(), + cast(F).getContents().size()); break; case MCFragment::FT_Relaxable: ++stats::EmittedRelaxableFragments; - OS << cast(F).getContents(); + OS << StringRef(cast(F).getContents().data(), + cast(F).getContents().size()); break; case MCFragment::FT_Fill: { @@ -691,7 +693,7 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm, case MCFragment::FT_LEB: { const MCLEBFragment &LF = cast(F); - OS << LF.getContents(); + OS << StringRef(LF.getContents().data(), LF.getContents().size()); break; } @@ -721,27 +723,27 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm, case MCFragment::FT_Dwarf: { const MCDwarfLineAddrFragment &OF = cast(F); - OS << OF.getContents(); + OS << StringRef(OF.getContents().data(), OF.getContents().size()); break; } case MCFragment::FT_DwarfFrame: { const MCDwarfCallFrameFragment &CF = cast(F); - OS << CF.getContents(); + OS << StringRef(CF.getContents().data(), CF.getContents().size()); break; } case MCFragment::FT_CVInlineLines: { const auto &OF = cast(F); - OS << OF.getContents(); + OS << StringRef(OF.getContents().data(), OF.getContents().size()); break; } case MCFragment::FT_CVDefRange: { const auto &DRF = cast(F); - OS << DRF.getContents(); + OS << StringRef(DRF.getContents().data(), DRF.getContents().size()); break; } case MCFragment::FT_PseudoProbe: { const MCPseudoProbeAddrFragment &PF = cast(F); - OS << PF.getContents(); + OS << StringRef(PF.getContents().data(), PF.getContents().size()); break; } } @@ -993,10 +995,11 @@ bool MCAssembler::relaxInstruction(MCRelaxableFragment &F) { // Encode the new instruction. F.setInst(Relaxed); - F.getFixups().clear(); - F.getContents().clear(); - getEmitter().encodeInstruction(Relaxed, F.getContents(), F.getFixups(), - *F.getSubtargetInfo()); + SmallVector Data; + SmallVector Fixups; + getEmitter().encodeInstruction(Relaxed, Data, Fixups, *F.getSubtargetInfo()); + F.setContents(Data); + F.setFixups(Fixups); return true; } @@ -1004,8 +1007,7 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) { const unsigned OldSize = static_cast(LF.getContents().size()); unsigned PadTo = OldSize; int64_t Value; - SmallVectorImpl &Data = LF.getContents(); - LF.getFixups().clear(); + LF.clearFixups(); // Use evaluateKnownAbsolute for Mach-O as a hack: .subsections_via_symbols // requires that .uleb128 A-B is foldable where A and B reside in different // fragments. This is used by __gcc_except_table. @@ -1026,17 +1028,18 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) { if (UseZeroPad) Value = 0; } - Data.clear(); - raw_svector_ostream OSE(Data); + uint8_t Data[16]; + size_t Size = 0; // The compiler can generate EH table assembly that is impossible to assemble // without either adding padding to an LEB fragment or adding extra padding // to a later alignment fragment. To accommodate such tables, relaxation can // only increase an LEB fragment size here, not decrease it. See PR35809. if (LF.isSigned()) - encodeSLEB128(Value, OSE, PadTo); + Size = encodeSLEB128(Value, Data, PadTo); else - encodeULEB128(Value, OSE, PadTo); - return OldSize != LF.getContents().size(); + Size = encodeULEB128(Value, Data, PadTo); + LF.setContents({reinterpret_cast(Data), Size}); + return OldSize != Size; } /// Check if the branch crosses the boundary. @@ -1106,19 +1109,19 @@ bool MCAssembler::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF) { return WasRelaxed; MCContext &Context = getContext(); - uint64_t OldSize = DF.getContents().size(); + auto OldSize = DF.getContents().size(); int64_t AddrDelta; bool Abs = DF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this); assert(Abs && "We created a line delta with an invalid expression"); (void)Abs; int64_t LineDelta; LineDelta = DF.getLineDelta(); - SmallVectorImpl &Data = DF.getContents(); - Data.clear(); - DF.getFixups().clear(); + SmallVector Data; MCDwarfLineAddr::encode(Context, getDWARFLinetableParams(), LineDelta, AddrDelta, Data); + DF.setContents(Data); + DF.clearFixups(); return OldSize != Data.size(); } @@ -1137,12 +1140,11 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCDwarfCallFrameFragment &DF) { return false; } - SmallVectorImpl &Data = DF.getContents(); - uint64_t OldSize = Data.size(); - Data.clear(); - DF.getFixups().clear(); - + auto OldSize = DF.getContents().size(); + SmallVector Data; MCDwarfFrameEmitter::encodeAdvanceLoc(Context, Value, Data); + DF.setContents(Data); + DF.clearFixups(); return OldSize != Data.size(); } @@ -1172,13 +1174,13 @@ bool MCAssembler::relaxPseudoProbeAddr(MCPseudoProbeAddrFragment &PF) { bool Abs = PF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this); assert(Abs && "We created a pseudo probe with an invalid expression"); (void)Abs; - SmallVectorImpl &Data = PF.getContents(); - Data.clear(); + SmallVector Data; raw_svector_ostream OSE(Data); - PF.getFixups().clear(); // AddrDelta is a signed integer encodeSLEB128(AddrDelta, OSE, OldSize); + PF.setContents(Data); + PF.clearFixups(); return OldSize != Data.size(); } diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp index e8f04271e84ce..5d7914396e09f 100644 --- a/llvm/lib/MC/MCCodeView.cpp +++ b/llvm/lib/MC/MCCodeView.cpp @@ -510,8 +510,7 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm, MCCVFunctionInfo *SiteInfo = getCVFunctionInfo(Frag.SiteFuncId); - SmallVectorImpl &Buffer = Frag.getContents(); - Buffer.clear(); // Clear old contents if we went through relaxation. + SmallVector Buffer; for (const MCCVLoc &Loc : Locs) { // Exit early if our line table would produce an oversized InlineSiteSym // record. Account for the ChangeCodeLength annotation emitted after the @@ -604,15 +603,14 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm, compressAnnotation(BinaryAnnotationsOpCode::ChangeCodeLength, Buffer); compressAnnotation(std::min(EndSymLength, LocAfterLength), Buffer); + Frag.setContents(Buffer); } void CodeViewContext::encodeDefRange(const MCAssembler &Asm, MCCVDefRangeFragment &Frag) { MCContext &Ctx = Asm.getContext(); - SmallVectorImpl &Contents = Frag.getContents(); - Contents.clear(); - SmallVectorImpl &Fixups = Frag.getFixups(); - Fixups.clear(); + SmallVector Contents; + SmallVector Fixups; raw_svector_ostream OS(Contents); // Compute all the sizes up front. @@ -692,4 +690,7 @@ void CodeViewContext::encodeDefRange(const MCAssembler &Asm, GapStartOffset += GapSize + RangeSize; } } + + Frag.setContents(Contents); + Frag.setFixups(Fixups); } diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp index 1fecde4be03cc..e84b91c5c41b0 100644 --- a/llvm/lib/MC/MCELFStreamer.cpp +++ b/llvm/lib/MC/MCELFStreamer.cpp @@ -448,11 +448,14 @@ void MCELFStreamer::emitInstToData(const MCInst &Inst, // Emit instruction directly into data fragment. size_t FixupStartIndex = DF->getFixups().size(); size_t CodeOffset = DF->getContents().size(); - Assembler.getEmitter().encodeInstruction(Inst, DF->getContents(), - DF->getFixups(), STI); - - auto Fixups = MutableArrayRef(DF->getFixups()).slice(FixupStartIndex); - for (auto &Fixup : Fixups) { + SmallVector Fixups; + Assembler.getEmitter().encodeInstruction(Inst, DF->getContentsForAppending(), + Fixups, STI); + DF->doneAppending(); + if (!Fixups.empty()) + DF->appendFixups(Fixups); + + for (auto &Fixup : MutableArrayRef(DF->getFixups()).slice(FixupStartIndex)) { Fixup.setOffset(Fixup.getOffset() + CodeOffset); if (Fixup.isLinkerRelaxable()) { DF->setLinkerRelaxable(); diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp index 7dd3b501757b7..5cf47e3325ff2 100644 --- a/llvm/lib/MC/MCFragment.cpp +++ b/llvm/lib/MC/MCFragment.cpp @@ -19,66 +19,22 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/raw_ostream.h" #include +#include #include using namespace llvm; +static_assert(std::is_trivially_destructible_v, + "fragment classes must be trivially destructible"); + MCFragment::MCFragment(FragmentType Kind, bool HasInstructions) : Kind(Kind), HasInstructions(HasInstructions), AlignToBundleEnd(false), LinkerRelaxable(false), AllowAutoPadding(false) {} -void MCFragment::destroy() { - switch (Kind) { - case FT_Align: - cast(this)->~MCAlignFragment(); - return; - case FT_Data: - cast(this)->~MCDataFragment(); - return; - case FT_Fill: - cast(this)->~MCFillFragment(); - return; - case FT_Nops: - cast(this)->~MCNopsFragment(); - return; - case FT_Relaxable: - cast(this)->~MCRelaxableFragment(); - return; - case FT_Org: - cast(this)->~MCOrgFragment(); - return; - case FT_Dwarf: - cast(this)->~MCDwarfLineAddrFragment(); - return; - case FT_DwarfFrame: - cast(this)->~MCDwarfCallFrameFragment(); - return; - case FT_LEB: - cast(this)->~MCLEBFragment(); - return; - case FT_BoundaryAlign: - cast(this)->~MCBoundaryAlignFragment(); - return; - case FT_SymbolId: - cast(this)->~MCSymbolIdFragment(); - return; - case FT_CVInlineLines: - cast(this)->~MCCVInlineLineTableFragment(); - return; - case FT_CVDefRange: - cast(this)->~MCCVDefRangeFragment(); - return; - case FT_PseudoProbe: - cast(this)->~MCPseudoProbeAddrFragment(); - return; - } -} - const MCSymbol *MCFragment::getAtom() const { return cast(Parent)->getAtom(LayoutOrder); } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void MCFragment::dump() const { raw_ostream &OS = errs(); @@ -131,7 +87,7 @@ LLVM_DUMP_METHOD void MCFragment::dump() const { const auto *F = cast(this); if (F->isLinkerRelaxable()) OS << " LinkerRelaxable"; - const SmallVectorImpl &Contents = F->getContents(); + auto Contents = F->getContents(); OS << " Size:" << Contents.size() << " ["; for (unsigned i = 0, e = Contents.size(); i != e; ++i) { if (i) OS << ","; diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 3b2bb594675f0..62b261a1268bc 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -63,28 +63,14 @@ void MCObjectStreamer::resolvePendingFixups() { PendingFixup.Fixup.setOffset(PendingFixup.Sym->getOffset() + PendingFixup.Fixup.getOffset()); - // If the location symbol to relocate is in MCEncodedFragmentWithFixups, + // If the location symbol to relocate is in MCEncodedFragment, // put the Fixup into location symbol's fragment. Otherwise // put into PendingFixup.DF MCFragment *SymFragment = PendingFixup.Sym->getFragment(); - switch (SymFragment->getKind()) { - case MCFragment::FT_Relaxable: - case MCFragment::FT_Dwarf: - case MCFragment::FT_PseudoProbe: - cast>(SymFragment) - ->getFixups() - .push_back(PendingFixup.Fixup); - break; - case MCFragment::FT_Data: - case MCFragment::FT_CVDefRange: - cast>(SymFragment) - ->getFixups() - .push_back(PendingFixup.Fixup); - break; - default: + if (auto *F = dyn_cast(SymFragment)) + F->addFixup(PendingFixup.Fixup); + else PendingFixup.DF->addFixup(PendingFixup.Fixup); - break; - } } PendingFixups.clear(); } @@ -398,10 +384,10 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst, getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI); auto CodeOffset = DF->getContents().size(); - for (MCFixup &Fixup : Fixups) { + for (MCFixup &Fixup : Fixups) Fixup.setOffset(Fixup.getOffset() + CodeOffset); - DF->addFixup(Fixup); - } + if (!Fixups.empty()) + DF->appendFixups(Fixups); DF->setHasInstructions(STI); DF->appendContents(Code); } @@ -414,8 +400,11 @@ void MCObjectStreamer::emitInstToFragment(const MCInst &Inst, getContext().allocFragment(Inst, STI); insert(IF); - getAssembler().getEmitter().encodeInstruction(Inst, IF->getContents(), - IF->getFixups(), STI); + SmallVector Fixups; + getAssembler().getEmitter().encodeInstruction( + Inst, IF->getContentsForAppending(), Fixups, STI); + IF->doneAppending(); + IF->appendFixups(Fixups); } #ifndef NDEBUG diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp index 1b766d3fad8b1..09d09e8f997e6 100644 --- a/llvm/lib/MC/MCSection.cpp +++ b/llvm/lib/MC/MCSection.cpp @@ -11,6 +11,7 @@ #include "llvm/Config/llvm-config.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -36,10 +37,13 @@ MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) { bool MCSection::hasEnded() const { return End && End->isInSection(); } MCSection::~MCSection() { + // If ~MCRelaxableFragment becomes trivial (no longer store a MCInst member), + // this dtor can be made empty. for (auto &[_, Chain] : Subsections) { for (MCFragment *X = Chain.Head, *Y; X; X = Y) { Y = X->Next; - X->destroy(); + if (auto *F = dyn_cast(X)) + F->~MCRelaxableFragment(); } } } @@ -88,3 +92,39 @@ LLVM_DUMP_METHOD void MCSection::dump( } } #endif + +void MCEncodedFragment::setContents(ArrayRef Contents) { + auto &S = getParent()->ContentStorage; + if (ContentStart + Contents.size() > ContentEnd) { + ContentStart = S.size(); + S.resize_for_overwrite(S.size() + Contents.size()); + } + ContentEnd = ContentStart + Contents.size(); + llvm::copy(Contents, S.begin() + ContentStart); +} + +void MCEncodedFragment::addFixup(MCFixup Fixup) { appendFixups({Fixup}); } + +void MCEncodedFragment::appendFixups(ArrayRef Fixups) { + auto &S = getParent()->FixupStorage; + if (LLVM_UNLIKELY(FixupEnd != S.size())) { + // Move the elements to the end. Reserve space to avoid invalidating + // S.begin()+I for `append`. + auto Size = FixupEnd - FixupStart; + auto I = std::exchange(FixupStart, S.size()); + S.reserve(S.size() + Size); + S.append(S.begin() + I, S.begin() + I + Size); + } + S.append(Fixups.begin(), Fixups.end()); + FixupEnd = S.size(); +} + +void MCEncodedFragment::setFixups(ArrayRef Fixups) { + auto &S = getParent()->FixupStorage; + if (FixupStart + Fixups.size() > FixupEnd) { + FixupStart = S.size(); + S.resize_for_overwrite(S.size() + Fixups.size()); + } + FixupEnd = FixupStart + Fixups.size(); + llvm::copy(Fixups, S.begin() + FixupStart); +} diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp index 2e671e0685c2e..eb7cb7d634311 100644 --- a/llvm/lib/MC/MachObjectWriter.cpp +++ b/llvm/lib/MC/MachObjectWriter.cpp @@ -807,8 +807,8 @@ uint64_t MachObjectWriter::writeObject() { MCSection *CGProfileSection = getContext().getMachOSection( "__LLVM", "__cg_profile", 0, SectionKind::getMetadata()); auto &Frag = cast(*CGProfileSection->begin()); - Frag.getContents().clear(); - raw_svector_ostream OS(Frag.getContents()); + Frag.clearContents(); + raw_svector_ostream OS(Frag.getContentsForAppending()); for (const MCObjectWriter::CGProfileEntry &CGPE : CGProfile) { uint32_t FromIndex = CGPE.From->getSymbol().getIndex(); uint32_t ToIndex = CGPE.To->getSymbol().getIndex(); @@ -816,6 +816,7 @@ uint64_t MachObjectWriter::writeObject() { support::endian::write(OS, ToIndex, W.Endian); support::endian::write(OS, CGPE.Count, W.Endian); } + Frag.doneAppending(); } unsigned NumSections = Asm.end() - Asm.begin(); diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp index a822a80b0af42..c2ef430984ed4 100644 --- a/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -1070,7 +1070,7 @@ uint64_t WinCOFFWriter::writeObject() { auto *Sec = getContext().getCOFFSection(".llvm_addrsig", COFF::IMAGE_SCN_LNK_REMOVE); auto *Frag = cast(Sec->curFragList()->Head); - raw_svector_ostream OS(Frag->getContents()); + raw_svector_ostream OS(Frag->getContentsForAppending()); for (const MCSymbol *S : OWriter.AddrsigSyms) { if (!S->isRegistered()) continue; @@ -1085,6 +1085,7 @@ uint64_t WinCOFFWriter::writeObject() { "executePostLayoutBinding!"); encodeULEB128(SectionMap[TargetSection]->Symbol->getIndex(), OS); } + Frag->doneAppending(); } // Create the contents of the .llvm.call-graph-profile section. @@ -1092,7 +1093,7 @@ uint64_t WinCOFFWriter::writeObject() { auto *Sec = getContext().getCOFFSection(".llvm.call-graph-profile", COFF::IMAGE_SCN_LNK_REMOVE); auto *Frag = cast(Sec->curFragList()->Head); - raw_svector_ostream OS(Frag->getContents()); + raw_svector_ostream OS(Frag->getContentsForAppending()); for (const auto &CGPE : OWriter.getCGProfile()) { uint32_t FromIndex = CGPE.From->getSymbol().getIndex(); uint32_t ToIndex = CGPE.To->getSymbol().getIndex(); @@ -1100,6 +1101,7 @@ uint64_t WinCOFFWriter::writeObject() { support::endian::write(OS, ToIndex, W.Endian); support::endian::write(OS, CGPE.Count, W.Endian); } + Frag->doneAppending(); } assignFileOffsets(); diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp index f7bb33d2790a1..64192ed6632de 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp @@ -297,9 +297,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, int64_t LineDelta = DF.getLineDelta(); const MCExpr &AddrDelta = DF.getAddrDelta(); - SmallVectorImpl &Data = DF.getContents(); - SmallVectorImpl &Fixups = DF.getFixups(); - size_t OldSize = Data.size(); + SmallVector Fixups; + size_t OldSize = DF.getContents().size(); int64_t Value; if (AddrDelta.evaluateAsAbsolute(Value, *Asm)) @@ -308,8 +307,7 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, assert(IsAbsolute && "CFA with invalid expression"); (void)IsAbsolute; - Data.clear(); - Fixups.clear(); + SmallVector Data; raw_svector_ostream OS(Data); // INT64_MAX is a signal that this is actually a DW_LNE_end_sequence. @@ -354,6 +352,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, OS << uint8_t(dwarf::DW_LNS_copy); } + DF.setContents(Data); + DF.setFixups(Fixups); WasRelaxed = OldSize != Data.size(); return true; } @@ -361,9 +361,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, bool &WasRelaxed) const { const MCExpr &AddrDelta = DF.getAddrDelta(); - SmallVectorImpl &Data = DF.getContents(); - SmallVectorImpl &Fixups = DF.getFixups(); - size_t OldSize = Data.size(); + SmallVector Fixups; + size_t OldSize = DF.getContents().size(); int64_t Value; if (AddrDelta.evaluateAsAbsolute(Value, *Asm)) @@ -372,14 +371,12 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, assert(IsAbsolute && "CFA with invalid expression"); (void)IsAbsolute; - Data.clear(); - Fixups.clear(); - raw_svector_ostream OS(Data); - assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 && "expected 1-byte alignment"); if (Value == 0) { - WasRelaxed = OldSize != Data.size(); + DF.clearContents(); + DF.clearFixups(); + WasRelaxed = OldSize != DF.getContents().size(); return true; } @@ -391,6 +388,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(FK))); }; + SmallVector Data; + raw_svector_ostream OS(Data); if (isUIntN(6, Value)) { OS << uint8_t(dwarf::DW_CFA_advance_loc); AddFixups(0, getRelocPairForSize(6)); @@ -409,6 +408,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, } else { llvm_unreachable("unsupported CFA encoding"); } + DF.setContents(Data); + DF.setFixups(Fixups); WasRelaxed = OldSize != Data.size(); return true; diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp index 952744b549811..75ef861d58a1c 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp @@ -274,17 +274,16 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, int64_t LineDelta = DF.getLineDelta(); const MCExpr &AddrDelta = DF.getAddrDelta(); - SmallVectorImpl &Data = DF.getContents(); - SmallVectorImpl &Fixups = DF.getFixups(); - size_t OldSize = Data.size(); + SmallVector Fixups; + size_t OldSize = DF.getContents().size(); int64_t Value; [[maybe_unused]] bool IsAbsolute = AddrDelta.evaluateKnownAbsolute(Value, *Asm); assert(IsAbsolute && "CFA with invalid expression"); - Data.clear(); Fixups.clear(); + SmallVector Data; raw_svector_ostream OS(Data); // INT64_MAX is a signal that this is actually a DW_LNE_end_sequence. @@ -329,6 +328,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, OS << uint8_t(dwarf::DW_LNS_copy); } + DF.setContents(Data); + DF.setFixups(Fixups); WasRelaxed = OldSize != Data.size(); return true; } @@ -336,9 +337,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF, bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, bool &WasRelaxed) const { const MCExpr &AddrDelta = DF.getAddrDelta(); - SmallVectorImpl &Data = DF.getContents(); - SmallVectorImpl &Fixups = DF.getFixups(); - size_t OldSize = Data.size(); + SmallVector Fixups; + size_t OldSize = DF.getContents().size(); int64_t Value; if (AddrDelta.evaluateAsAbsolute(Value, *Asm)) @@ -347,14 +347,12 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, AddrDelta.evaluateKnownAbsolute(Value, *Asm); assert(IsAbsolute && "CFA with invalid expression"); - Data.clear(); - Fixups.clear(); - raw_svector_ostream OS(Data); - assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 && "expected 1-byte alignment"); if (Value == 0) { - WasRelaxed = OldSize != Data.size(); + DF.clearContents(); + DF.clearFixups(); + WasRelaxed = OldSize != DF.getContents().size(); return true; } @@ -365,6 +363,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(Fixup))); }; + SmallVector Data; + raw_svector_ostream OS(Data); if (isUIntN(6, Value)) { OS << uint8_t(dwarf::DW_CFA_advance_loc); AddFixups(0, {ELF::R_RISCV_SET6, ELF::R_RISCV_SUB6}); @@ -383,6 +383,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF, } else { llvm_unreachable("unsupported CFA encoding"); } + DF.setContents(Data); + DF.setFixups(Fixups); WasRelaxed = OldSize != Data.size(); return true; diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index 361d7b488d048..77e2011361162 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -847,7 +847,7 @@ bool X86AsmBackend::padInstructionViaRelaxation(MCRelaxableFragment &RF, return false; RF.setInst(Relaxed); RF.setContents(Code); - RF.getFixups() = Fixups; + RF.setFixups(Fixups); RemainingSize -= Delta; return true; }