Dyn project

bolt/include/bolt/Core/BinaryContext.h

@@ -615,6 +615,9 @@ class BinaryContext {
   /// Number of functions with profile information
   uint64_t NumProfiledFuncs{0};
 
+  /// Number of functions with stale profile information
+  uint64_t NumStaleProfileFuncs{0};
+
   /// Number of objects in profile whose profile was ignored.
   uint64_t NumUnusedProfiledObjects{0};
 

bolt/include/bolt/Core/BinarySection.h

@@ -43,6 +43,9 @@ class BinaryData;
 class BinarySection {
   friend class BinaryContext;
 
+  /// Count the number of sections created.
+  static uint64_t Count;
+
   BinaryContext &BC;           // Owning BinaryContext
   std::string Name;            // Section name
   const SectionRef Section;    // SectionRef (may be null)
@@ -86,6 +89,7 @@ class BinarySection {
   uint64_t OutputSize{0};          // Section size in the rewritten binary.
   uint64_t OutputFileOffset{0};    // File offset in the rewritten binary file.
   StringRef OutputContents;        // Rewritten section contents.
+  const uint64_t SectionNumber;    // Order in which the section was created.
   unsigned SectionID{-1u};         // Unique ID used for address mapping.
                                    // Set by ExecutableFileMemoryManager.
   uint32_t Index{0};               // Section index in the output file.
@@ -147,13 +151,14 @@ class BinarySection {
         Size(Section.getSize()), Alignment(Section.getAlignment()),
         ELFType(Section.getELFType()), ELFFlags(Section.getELFFlags()),
         Relocations(Section.Relocations),
-        PendingRelocations(Section.PendingRelocations), OutputName(Name) {}
+        PendingRelocations(Section.PendingRelocations), OutputName(Name),
+        SectionNumber(++Count) {}
 
   BinarySection(BinaryContext &BC, SectionRef Section)
       : BC(BC), Name(getName(Section)), Section(Section),
         Contents(getContents(Section)), Address(Section.getAddress()),
         Size(Section.getSize()), Alignment(Section.getAlignment()),
-        OutputName(Name) {
+        OutputName(Name), SectionNumber(++Count) {
     if (isELF()) {
       ELFType = ELFSectionRef(Section).getType();
       ELFFlags = ELFSectionRef(Section).getFlags();
@@ -173,7 +178,7 @@ class BinarySection {
         Contents(reinterpret_cast<const char *>(Data), Data ? Size : 0),
         Address(0), Size(Size), Alignment(Alignment), ELFType(ELFType),
         ELFFlags(ELFFlags), IsFinalized(true), OutputName(Name),
-        OutputSize(Size), OutputContents(Contents) {
+        OutputSize(Size), OutputContents(Contents), SectionNumber(++Count) {
     assert(Alignment > 0 && "section alignment must be > 0");
   }
 
@@ -207,10 +212,34 @@ class BinarySection {
 
   // Order sections by their immutable properties.
   bool operator<(const BinarySection &Other) const {
-    return (getAddress() < Other.getAddress() ||
-            (getAddress() == Other.getAddress() &&
-             (getSize() < Other.getSize() ||
-              (getSize() == Other.getSize() && getName() < Other.getName()))));
+    // Allocatable before non-allocatable.
+    if (isAllocatable() != Other.isAllocatable())
+      return isAllocatable() > Other.isAllocatable();
+
+    // Input sections take precedence.
+    if (hasSectionRef() != Other.hasSectionRef())
+      return hasSectionRef() > Other.hasSectionRef();
+
+    // Compare allocatable input sections by their address.
+    if (getAddress() != Other.getAddress())
+      return getAddress() < Other.getAddress();
+    if (getAddress() && getSize() != Other.getSize())
+      return getSize() < Other.getSize();
+
+    // Code before data.
+    if (isText() != Other.isText())
+      return isText() > Other.isText();
+
+    // Read-only before writable.
+    if (isReadOnly() != Other.isReadOnly())
+      return isReadOnly() > Other.isReadOnly();
+
+    // BSS at the end.
+    if (isBSS() != Other.isBSS())
+      return isBSS() < Other.isBSS();
+
+    // Otherwise, preserve the order of creation.
+    return SectionNumber < Other.SectionNumber;
   }
 
   ///
@@ -228,13 +257,13 @@ class BinarySection {
   bool isText() const {
     if (isELF())
       return (ELFFlags & ELF::SHF_EXECINSTR);
-    return getSectionRef().isText();
+    return hasSectionRef() && getSectionRef().isText();
   }
   bool isData() const {
     if (isELF())
       return (ELFType == ELF::SHT_PROGBITS &&
               (ELFFlags & (ELF::SHF_ALLOC | ELF::SHF_WRITE)));
-    return getSectionRef().isData();
+    return hasSectionRef() && getSectionRef().isData();
   }
   bool isBSS() const {
     return (ELFType == ELF::SHT_NOBITS &&

bolt/include/bolt/Core/DebugData.h

@@ -330,8 +330,8 @@ class DebugAddrWriter {
 protected:
   class AddressForDWOCU {
   public:
-    AddressToIndexMap::iterator find(uint64_t Adddress) {
-      return AddressToIndex.find(Adddress);
+    AddressToIndexMap::iterator find(uint64_t Address) {
+      return AddressToIndex.find(Address);
     }
     AddressToIndexMap::iterator end() { return AddressToIndex.end(); }
     AddressToIndexMap::iterator begin() { return AddressToIndex.begin(); }

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1431,7 +1431,7 @@ class MCPlusBuilder {
     return false;
   }
 
-  /// Store \p Target absolute adddress to \p RegName
+  /// Store \p Target absolute address to \p RegName
   virtual InstructionListType materializeAddress(const MCSymbol *Target,
                                                  MCContext *Ctx,
                                                  MCPhysReg RegName,

bolt/include/bolt/Profile/DataAggregator.h

@@ -311,6 +311,10 @@ class DataAggregator : public DataReader {
   /// Consume the entire line
   void consumeRestOfLine();
 
+  /// True if the next token in the parsing buffer is a new line, but don't
+  /// consume it (peek only).
+  bool checkNewLine();
+
   /// Parse a single LBR entry as output by perf script -Fbrstack
   ErrorOr<LBREntry> parseLBREntry();
 

bolt/lib/Core/BinaryEmitter.cpp

@@ -211,8 +211,6 @@ void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
   }
 
   emitDataSections(OrgSecPrefix);
-
-  Streamer.emitLabel(BC.Ctx->getOrCreateSymbol("_end"));
 }
 
 void BinaryEmitter::emitFunctions() {

bolt/lib/Core/BinarySection.cpp

@@ -26,6 +26,8 @@ extern cl::opt<bool> PrintRelocations;
 extern cl::opt<bool> HotData;
 } // namespace opts
 
+uint64_t BinarySection::Count = 0;
+
 bool BinarySection::isELF() const { return BC.isELF(); }
 
 bool BinarySection::isMachO() const { return BC.isMachO(); }

bolt/lib/Passes/BinaryPasses.cpp

@@ -417,10 +417,14 @@ void ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) {
   ParallelUtilities::runOnEachFunction(
       BC, ParallelUtilities::SchedulingPolicy::SP_BB_LINEAR, WorkFun, SkipFunc,
       "ReorderBasicBlocks");
+  const size_t NumAllProfiledFunctions =
+      BC.NumProfiledFuncs + BC.NumStaleProfileFuncs;
 
   outs() << "BOLT-INFO: basic block reordering modified layout of "
-         << format("%zu (%.2lf%%) functions\n",
+         << format("%zu functions (%.2lf%% of profiled, %.2lf%% of total)\n",
                    ModifiedFuncCount.load(std::memory_order_relaxed),
+                   100.0 * ModifiedFuncCount.load(std::memory_order_relaxed) /
+                       NumAllProfiledFunctions,
                    100.0 * ModifiedFuncCount.load(std::memory_order_relaxed) /
                        BC.getBinaryFunctions().size());
 
@@ -890,7 +894,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) {
 
       // Annotate it, so "isCall" returns true for this jcc
       MIB->setConditionalTailCall(*CondBranch);
-      // Add info abount the conditional tail call frequency, otherwise this
+      // Add info about the conditional tail call frequency, otherwise this
       // info will be lost when we delete the associated BranchInfo entry
       auto &CTCAnnotation =
           MIB->getOrCreateAnnotationAs<uint64_t>(*CondBranch, "CTCTakenCount");
@@ -1383,6 +1387,7 @@ void PrintProgramStats::runOnFunctions(BinaryContext &BC) {
     }
   }
   BC.NumProfiledFuncs = ProfiledFunctions.size();
+  BC.NumStaleProfileFuncs = NumStaleProfileFunctions;
 
   const size_t NumAllProfiledFunctions =
       ProfiledFunctions.size() + NumStaleProfileFunctions;

bolt/lib/Passes/IndirectCallPromotion.cpp

@@ -416,15 +416,15 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
 
   ++TotalIndexBasedCandidates;
 
-  auto ErrorOrMemAccesssProfile =
+  auto ErrorOrMemAccessProfile =
       BC.MIB->tryGetAnnotationAs<MemoryAccessProfile>(*MemLocInstr,
                                                       "MemoryAccessProfile");
-  if (!ErrorOrMemAccesssProfile) {
+  if (!ErrorOrMemAccessProfile) {
     DEBUG_VERBOSE(1, dbgs()
                          << "BOLT-INFO: ICP no memory profiling data found\n");
     return JumpTableInfoType();
   }
-  MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccesssProfile.get();
+  MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccessProfile.get();
 
   uint64_t ArrayStart;
   if (DispExpr) {
@@ -670,15 +670,15 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms(
   });
 
   // Try to get value profiling data for the method load instruction.
-  auto ErrorOrMemAccesssProfile =
+  auto ErrorOrMemAccessProfile =
       BC.MIB->tryGetAnnotationAs<MemoryAccessProfile>(*MethodFetchInsns.back(),
                                                       "MemoryAccessProfile");
-  if (!ErrorOrMemAccesssProfile) {
+  if (!ErrorOrMemAccessProfile) {
     DEBUG_VERBOSE(1, dbgs()
                          << "BOLT-INFO: ICP no memory profiling data found\n");
     return MethodInfoType();
   }
-  MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccesssProfile.get();
+  MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccessProfile.get();
 
   // Find the vtable that each method belongs to.
   std::map<const MCSymbol *, uint64_t> MethodToVtable;

bolt/lib/Passes/ReorderData.cpp

@@ -182,14 +182,14 @@ void ReorderData::assignMemData(BinaryContext &BC) {
 
     for (const BinaryBasicBlock &BB : BF) {
       for (const MCInst &Inst : BB) {
-        auto ErrorOrMemAccesssProfile =
+        auto ErrorOrMemAccessProfile =
             BC.MIB->tryGetAnnotationAs<MemoryAccessProfile>(
                 Inst, "MemoryAccessProfile");
-        if (!ErrorOrMemAccesssProfile)
+        if (!ErrorOrMemAccessProfile)
           continue;
 
         const MemoryAccessProfile &MemAccessProfile =
-            ErrorOrMemAccesssProfile.get();
+            ErrorOrMemAccessProfile.get();
         for (const AddressAccess &AccessInfo :
              MemAccessProfile.AddressAccessInfo) {
           if (BinaryData *BD = AccessInfo.MemoryObject) {
@@ -238,14 +238,14 @@ ReorderData::sortedByFunc(BinaryContext &BC, const BinarySection &Section,
         continue;
 
       for (const MCInst &Inst : BB) {
-        auto ErrorOrMemAccesssProfile =
+        auto ErrorOrMemAccessProfile =
             BC.MIB->tryGetAnnotationAs<MemoryAccessProfile>(
                 Inst, "MemoryAccessProfile");
-        if (!ErrorOrMemAccesssProfile)
+        if (!ErrorOrMemAccessProfile)
           continue;
 
         const MemoryAccessProfile &MemAccessProfile =
-            ErrorOrMemAccesssProfile.get();
+            ErrorOrMemAccessProfile.get();
         for (const AddressAccess &AccessInfo :
              MemAccessProfile.AddressAccessInfo) {
           if (AccessInfo.MemoryObject)

bolt/lib/Passes/RetpolineInsertion.cpp

@@ -44,19 +44,17 @@ RetpolineLfence("retpoline-lfence",
   cl::Hidden,
   cl::cat(BoltCategory));
 
-cl::opt<RetpolineInsertion::AvailabilityOptions>
-R11Availability("r11-availability",
-  cl::desc("determine the availablity of r11 before indirect branches"),
-  cl::init(RetpolineInsertion::AvailabilityOptions::NEVER),
-  cl::values(
-    clEnumValN(RetpolineInsertion::AvailabilityOptions::NEVER,
-      "never", "r11 not available"),
-    clEnumValN(RetpolineInsertion::AvailabilityOptions::ALWAYS,
-      "always", "r11 avaialable before calls and jumps"),
-    clEnumValN(RetpolineInsertion::AvailabilityOptions::ABI,
-      "abi", "r11 avaialable before calls but not before jumps")),
-  cl::ZeroOrMore,
-  cl::cat(BoltCategory));
+cl::opt<RetpolineInsertion::AvailabilityOptions> R11Availability(
+    "r11-availability",
+    cl::desc("determine the availability of r11 before indirect branches"),
+    cl::init(RetpolineInsertion::AvailabilityOptions::NEVER),
+    cl::values(clEnumValN(RetpolineInsertion::AvailabilityOptions::NEVER,
+                          "never", "r11 not available"),
+               clEnumValN(RetpolineInsertion::AvailabilityOptions::ALWAYS,
+                          "always", "r11 avaialable before calls and jumps"),
+               clEnumValN(RetpolineInsertion::AvailabilityOptions::ABI, "abi",
+                          "r11 avaialable before calls but not before jumps")),
+    cl::ZeroOrMore, cl::cat(BoltCategory));
 
 } // namespace opts
 

bolt/lib/Profile/DataAggregator.cpp

@@ -1056,6 +1056,10 @@ void DataAggregator::consumeRestOfLine() {
   Line += 1;
 }
 
+bool DataAggregator::checkNewLine() {
+  return ParsingBuf[0] == '\n';
+}
+
 ErrorOr<DataAggregator::PerfBranchSample> DataAggregator::parseBranchSample() {
   PerfBranchSample Res;
 
@@ -2147,7 +2151,8 @@ DataAggregator::parseNameBuildIDPair() {
 
   // If one of the strings is missing, don't issue a parsing error, but still
   // do not return a value.
-  if (ParsingBuf[0] == '\n')
+  consumeAllRemainingFS();
+  if (checkNewLine())
     return NoneType();
 
   ErrorOr<StringRef> NameStr = parseString(FieldSeparator, true);