Manually merged main:5821351343eb into amd-gfx:ca668accdd

Local branch amd-gfx Merged main:cce35994bb72 into amd-gfx:b53c56f743f7
Remote branch main 5821351 [clang][Interp] Implement __builtin_fpclassify

Author: mariusz-sikora-at-amd

bolt/include/bolt/Passes/ReorderFunctions.h

@@ -32,6 +32,7 @@ class ReorderFunctions : public BinaryFunctionPass {
     RT_EXEC_COUNT,
     RT_HFSORT,
     RT_HFSORT_PLUS,
+    RT_CDS,
     RT_PETTIS_HANSEN,
     RT_RANDOM,
     RT_USER

bolt/lib/Core/BinaryFunction.cpp

@@ -2305,6 +2305,12 @@ void BinaryFunction::removeConditionalTailCalls() {
 
     // This branch is no longer a conditional tail call.
     BC.MIB->unsetConditionalTailCall(*CTCInstr);
+
+    // Move offset from CTCInstr to TailCallInstr.
+    if (std::optional<uint32_t> Offset = BC.MIB->getOffset(*CTCInstr)) {
+      BC.MIB->setOffset(TailCallInstr, *Offset);
+      BC.MIB->clearOffset(*CTCInstr);
+    }
   }
 
   insertBasicBlocks(std::prev(end()), std::move(NewBlocks),

bolt/lib/Passes/ReorderFunctions.cpp

@@ -15,6 +15,7 @@
 #include "bolt/Utils/Utils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils/CodeLayout.h"
 #include <fstream>
 
 #define DEBUG_TYPE "hfsort"
@@ -41,6 +42,8 @@ cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions(
                           "use hfsort algorithm"),
                clEnumValN(bolt::ReorderFunctions::RT_HFSORT_PLUS, "hfsort+",
                           "use hfsort+ algorithm"),
+               clEnumValN(bolt::ReorderFunctions::RT_CDS, "cds",
+                          "use cache-directed sort"),
                clEnumValN(bolt::ReorderFunctions::RT_PETTIS_HANSEN,
                           "pettis-hansen", "use Pettis-Hansen algorithm"),
                clEnumValN(bolt::ReorderFunctions::RT_RANDOM, "random",
@@ -309,6 +312,37 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
   case RT_HFSORT_PLUS:
     Clusters = hfsortPlus(Cg);
     break;
+  case RT_CDS: {
+    // It is required that the sum of incoming arc weights is not greater
+    // than the number of samples for every function. Ensuring the call graph
+    // obeys the property before running the algorithm.
+    Cg.adjustArcWeights();
+
+    // Initialize CFG nodes and their data
+    std::vector<uint64_t> FuncSizes;
+    std::vector<uint64_t> FuncCounts;
+    using JumpT = std::pair<uint64_t, uint64_t>;
+    std::vector<std::pair<JumpT, uint64_t>> CallCounts;
+    std::vector<uint64_t> CallOffsets;
+    for (NodeId F = 0; F < Cg.numNodes(); ++F) {
+      FuncSizes.push_back(Cg.size(F));
+      FuncCounts.push_back(Cg.samples(F));
+      for (NodeId Succ : Cg.successors(F)) {
+        const Arc &Arc = *Cg.findArc(F, Succ);
+        auto It = std::make_pair(F, Succ);
+        CallCounts.push_back(std::make_pair(It, Arc.weight()));
+        CallOffsets.push_back(uint64_t(Arc.avgCallOffset()));
+      }
+    }
+
+    // Run the layout algorithm.
+    std::vector<uint64_t> Result =
+        applyCDSLayout(FuncSizes, FuncCounts, CallCounts, CallOffsets);
+
+    // Create a single cluster from the computed order of hot functions.
+    std::vector<CallGraph::NodeId> NodeOrder(Result.begin(), Result.end());
+    Clusters.emplace_back(Cluster(NodeOrder, Cg));
+  } break;
   case RT_PETTIS_HANSEN:
     Clusters = pettisAndHansen(Cg);
     break;

bolt/lib/Profile/DataAggregator.cpp

@@ -1479,13 +1479,10 @@ std::error_code DataAggregator::parseBranchEvents() {
     NumTraces += parseLBRSample(Sample, NeedsSkylakeFix);
   }
 
-  for (const auto &LBR : BranchLBRs) {
-    const Trace &Trace = LBR.first;
-    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Trace.From))
-      BF->setHasProfileAvailable();
-    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Trace.To))
-      BF->setHasProfileAvailable();
-  }
+  for (const Trace &Trace : llvm::make_first_range(BranchLBRs))
+    for (const uint64_t Addr : {Trace.From, Trace.To})
+      if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Addr))
+        BF->setHasProfileAvailable();
 
   auto printColored = [](raw_ostream &OS, float Percent, float T1, float T2) {
     OS << " (";
@@ -1721,12 +1718,9 @@ std::error_code DataAggregator::parsePreAggregatedLBRSamples() {
     if (std::error_code EC = AggrEntry.getError())
       return EC;
 
-    if (BinaryFunction *BF =
-            getBinaryFunctionContainingAddress(AggrEntry->From.Offset))
-      BF->setHasProfileAvailable();
-    if (BinaryFunction *BF =
-            getBinaryFunctionContainingAddress(AggrEntry->To.Offset))
-      BF->setHasProfileAvailable();
+    for (const uint64_t Addr : {AggrEntry->From.Offset, AggrEntry->To.Offset})
+      if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Addr))
+        BF->setHasProfileAvailable();
 
     AggregatedLBRs.emplace_back(std::move(AggrEntry.get()));
   }

bolt/lib/Profile/YAMLProfileReader.cpp

@@ -344,8 +344,10 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
         continue;
 
       yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue();
-      if (profileMatches(YamlBF, Function))
+      if (profileMatches(YamlBF, Function)) {
         matchProfileToFunction(YamlBF, Function);
+        break;
+      }
     }
   }
 

bolt/lib/Rewrite/DWARFRewriter.cpp

@@ -839,7 +839,7 @@ void DWARFRewriter::updateUnitDebugInfo(
     case dwarf::DW_TAG_inlined_subroutine:
     case dwarf::DW_TAG_try_block:
     case dwarf::DW_TAG_catch_block: {
-      uint64_t RangesSectionOffset = RangesSectionWriter.getEmptyRangesOffset();
+      uint64_t RangesSectionOffset = 0;
       Expected<DWARFAddressRangesVector> RangesOrError =
           getDIEAddressRanges(*Die, Unit);
       const BinaryFunction *Function =
@@ -865,6 +865,10 @@ void DWARFRewriter::updateUnitDebugInfo(
         }
       } else if (!RangesOrError) {
         consumeError(RangesOrError.takeError());
+      } else {
+        OutputRanges.push_back({0, !RangesOrError->empty()
+                                       ? RangesOrError.get().front().HighPC
+                                       : 0});
       }
       DIEValue LowPCVal = Die->findAttribute(dwarf::DW_AT_low_pc);
       DIEValue HighPCVal = Die->findAttribute(dwarf::DW_AT_high_pc);