Merged main:e9c4dc18bc9d into amd-gfx:162dd7710cc4

Local branch amd-gfx 162dd77 Merged main:a87638596a78 into amd-gfx:d1338c27690c
Remote branch main e9c4dc1 Revert "[AMDGPU] Use `S_CSELECT` for uniform i1 ext (llvm#69703)"

Author: SC llvm team

.github/workflows/docs.yml

@@ -17,30 +17,31 @@ on:
       - 'clang/docs/**'
       - 'clang-tools-extra/docs/**'
       - 'lldb/docs/**'
+      - 'libunwind/docs/**'
+      - 'libcxx/docs/**'
+      - 'libc/docs/**'
+      - 'lld/docs/**'
   pull_request:
     paths:
       - 'llvm/docs/**'
       - 'clang/docs/**'
       - 'clang-tools-extra/docs/**'
       - 'lldb/docs/**'
+      - 'libunwind/docs/**'
+      - 'libcxx/docs/**'
+      - 'libc/docs/**'
+      - 'lld/docs/**'
 
 jobs:
   check-docs-build:
     name: "Test documentation build"
     runs-on: ubuntu-latest
     steps:
-      # Fetch all the commits in a pull request + 1 so that the
-      # docs-changed-subprojects step won't pull them in itself in an extremely
-      # slow manner.
-      - name: Calculate number of commits to fetch (PR)
-        if: ${{ github.event_name == 'pull_request' }} 
-        run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
-      - name: Fetch LLVM sources (PR)
-        if: ${{ github.event_name == 'pull_request' }}
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: ${{ env.PR_FETCH_DEPTH }}
-      - name: Fetch LLVM sources (push)
+      # Don't fetch before checking for file changes to force the file changes
+      # action to use the Github API in pull requests. If it's a push to a
+      # branch we can't use the Github API to get the diff, so we need to have
+      # a local checkout beforehand.
+      - name: Fetch LLVM sources (Push)
         if: ${{ github.event_name == 'push' }}
         uses: actions/checkout@v4
         with:
@@ -58,6 +59,19 @@ jobs:
               - 'clang-tools-extra/docs/**'
             lldb:
               - 'lldb/docs/**'
+            libunwind:
+              - 'libunwind/docs/**'
+            libcxx:
+              - 'libcxx/docs/**'
+            libc:
+              - 'libc/docs/**'
+            lld:
+              - 'lld/docs/**'
+      - name: Fetch LLVM sources (PR)
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
       - name: Setup Python env
         uses: actions/setup-python@v4
         with:
@@ -91,4 +105,24 @@ jobs:
         run: |
           cmake -B lldb-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;lldb" -DLLVM_ENABLE_SPHINX=ON ./llvm
           TZ=UTC ninja -C lldb-build docs-lldb-html docs-lldb-man
+      - name: Build libunwind docs
+        if: steps.docs-changed-subprojects.outputs.libunwind_any_changed == 'true'
+        run: |
+          cmake -B libunwind-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libunwind" -DLLVM_ENABLE_SPHINX=ON ./runtimes
+          TZ=UTC ninja -C libunwind-build docs-libunwind-html
+      - name: Build libcxx docs
+        if: steps.docs-changed-subprojects.outputs.libcxx_any_changed == 'true'
+        run: |
+          cmake -B libcxx-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libcxxabi;libcxx" -DLLVM_ENABLE_SPHINX=ON ./runtimes
+          TZ=UTC ninja -C libcxx-build docs-libcxx-html
+      - name: Build libc docs
+        if: steps.docs-changed-subprojects.outputs.libc_any_changed == 'true'
+        run: |
+          cmake -B libc-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libc" -DLLVM_ENABLE_SPHINX=ON ./runtimes
+          TZ=UTC ninja -C docs-libc-html
+      - name: Build LLD docs
+        if: steps.docs-changed-subprojects.outputs.lld_any_changed == 'true'
+        run: |
+          cmake -B lld-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="lld" -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C lld-build docs-lld-html
 

bolt/include/bolt/Core/AddressMap.h

@@ -6,18 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Helper class to create a mapping from input to output addresses needed for
-// updating debugging symbols and BAT. We emit an MCSection containing
-// <Input address, Output MCSymbol> pairs to the object file and JITLink will
-// transform this in <Input address, Output address> pairs. The linker output
-// can then be parsed and used to establish the mapping.
+// This file contains the declaration of the AddressMap class used for looking
+// up addresses in the output object.
 //
 //===----------------------------------------------------------------------===//
-//
+
 #ifndef BOLT_CORE_ADDRESS_MAP_H
 #define BOLT_CORE_ADDRESS_MAP_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCSymbol.h"
 
 #include <optional>
 #include <unordered_map>
@@ -30,26 +28,48 @@ namespace bolt {
 
 class BinaryContext;
 
+/// Helper class to create a mapping from input entities to output addresses
+/// needed for updating debugging symbols and BAT. We emit a section containing
+/// <Input entity, Output MCSymbol> pairs to the object file and JITLink will
+/// transform this in <Input entity, Output address> pairs. The linker output
+/// can then be parsed and used to establish the mapping.
+///
+/// The entities that can be mapped to output address are input addresses and
+/// labels (MCSymbol). Input addresses support one-to-many mapping.
 class AddressMap {
-  using MapTy = std::unordered_multimap<uint64_t, uint64_t>;
-  MapTy Map;
+  static const char *const AddressSectionName;
+  static const char *const LabelSectionName;
 
-public:
-  static const char *const SectionName;
+  /// Map multiple <input address> to <output address>.
+  using Addr2AddrMapTy = std::unordered_multimap<uint64_t, uint64_t>;
+  Addr2AddrMapTy Address2AddressMap;
 
+  /// Map MCSymbol to its output address. Normally used for temp symbols that
+  /// are not updated by the linker.
+  using Label2AddrMapTy = DenseMap<const MCSymbol *, uint64_t>;
+  Label2AddrMapTy Label2AddrMap;
+
+public:
   static void emit(MCStreamer &Streamer, BinaryContext &BC);
-  static AddressMap parse(StringRef Buffer, const BinaryContext &BC);
+  static std::optional<AddressMap> parse(BinaryContext &BC);
 
   std::optional<uint64_t> lookup(uint64_t InputAddress) const {
-    auto It = Map.find(InputAddress);
-    if (It != Map.end())
+    auto It = Address2AddressMap.find(InputAddress);
+    if (It != Address2AddressMap.end())
+      return It->second;
+    return std::nullopt;
+  }
+
+  std::optional<uint64_t> lookup(const MCSymbol *Symbol) const {
+    auto It = Label2AddrMap.find(Symbol);
+    if (It != Label2AddrMap.end())
       return It->second;
     return std::nullopt;
   }
 
-  std::pair<MapTy::const_iterator, MapTy::const_iterator>
+  std::pair<Addr2AddrMapTy::const_iterator, Addr2AddrMapTy::const_iterator>
   lookupAll(uint64_t InputAddress) const {
-    return Map.equal_range(InputAddress);
+    return Address2AddressMap.equal_range(InputAddress);
   }
 };
 

bolt/lib/Core/AddressMap.cpp

@@ -1,22 +1,44 @@
+//===- bolt/Core/AddressMap.cpp - Input-output Address Map ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #include "bolt/Core/AddressMap.h"
 #include "bolt/Core/BinaryContext.h"
 #include "bolt/Core/BinaryFunction.h"
+#include "bolt/Core/BinarySection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Support/DataExtractor.h"
 
 namespace llvm {
 namespace bolt {
 
-const char *const AddressMap::SectionName = ".bolt.address_map";
+const char *const AddressMap::AddressSectionName = ".bolt.addr2addr_map";
+const char *const AddressMap::LabelSectionName = ".bolt.label2addr_map";
 
-static void emitLabel(MCStreamer &Streamer, uint64_t InputAddress,
-                      const MCSymbol *OutputLabel) {
+static void emitAddress(MCStreamer &Streamer, uint64_t InputAddress,
+                        const MCSymbol *OutputLabel) {
   Streamer.emitIntValue(InputAddress, 8);
   Streamer.emitSymbolValue(OutputLabel, 8);
 }
 
+static void emitLabel(MCStreamer &Streamer, const MCSymbol *OutputLabel) {
+  Streamer.emitIntValue(reinterpret_cast<uint64_t>(OutputLabel), 8);
+  Streamer.emitSymbolValue(OutputLabel, 8);
+}
+
 void AddressMap::emit(MCStreamer &Streamer, BinaryContext &BC) {
-  Streamer.switchSection(BC.getDataSection(SectionName));
+  // Mark map sections as link-only to avoid allocation in the output file.
+  const unsigned Flags = BinarySection::getFlags(/*IsReadOnly*/ true,
+                                                 /*IsText*/ false,
+                                                 /*IsAllocatable*/ true);
+  BC.registerOrUpdateSection(AddressSectionName, ELF::SHT_PROGBITS, Flags)
+      .setLinkOnly();
+  BC.registerOrUpdateSection(LabelSectionName, ELF::SHT_PROGBITS, Flags)
+      .setLinkOnly();
 
   for (const auto &[BFAddress, BF] : BC.getBinaryFunctions()) {
     if (!BF.requiresAddressMap())
@@ -26,37 +48,69 @@ void AddressMap::emit(MCStreamer &Streamer, BinaryContext &BC) {
       if (!BB.getLabel()->isDefined())
         continue;
 
-      emitLabel(Streamer, BFAddress + BB.getInputAddressRange().first,
-                BB.getLabel());
+      Streamer.switchSection(BC.getDataSection(LabelSectionName));
+      emitLabel(Streamer, BB.getLabel());
 
       if (!BB.hasLocSyms())
         continue;
 
+      Streamer.switchSection(BC.getDataSection(AddressSectionName));
       for (auto [Offset, Symbol] : BB.getLocSyms())
-        emitLabel(Streamer, BFAddress + Offset, Symbol);
+        emitAddress(Streamer, BFAddress + Offset, Symbol);
     }
   }
 }
 
-AddressMap AddressMap::parse(StringRef Buffer, const BinaryContext &BC) {
-  const auto EntrySize = 2 * BC.AsmInfo->getCodePointerSize();
-  assert(Buffer.size() % EntrySize == 0 && "Unexpected address map size");
+std::optional<AddressMap> AddressMap::parse(BinaryContext &BC) {
+  auto AddressMapSection = BC.getUniqueSectionByName(AddressSectionName);
+  auto LabelMapSection = BC.getUniqueSectionByName(LabelSectionName);
 
-  DataExtractor DE(Buffer, BC.AsmInfo->isLittleEndian(),
-                   BC.AsmInfo->getCodePointerSize());
-  DataExtractor::Cursor Cursor(0);
+  if (!AddressMapSection && !LabelMapSection)
+    return std::nullopt;
 
   AddressMap Parsed;
-  Parsed.Map.reserve(Buffer.size() / EntrySize);
 
-  while (Cursor && !DE.eof(Cursor)) {
-    const auto Input = DE.getAddress(Cursor);
-    const auto Output = DE.getAddress(Cursor);
-    if (!Parsed.Map.count(Input))
-      Parsed.Map.insert({Input, Output});
+  const size_t EntrySize = 2 * BC.AsmInfo->getCodePointerSize();
+  auto parseSection =
+      [&](BinarySection &Section,
+          function_ref<void(uint64_t, uint64_t)> InsertCallback) {
+        StringRef Buffer = Section.getOutputContents();
+        assert(Buffer.size() % EntrySize == 0 && "Unexpected address map size");
+
+        DataExtractor DE(Buffer, BC.AsmInfo->isLittleEndian(),
+                         BC.AsmInfo->getCodePointerSize());
+        DataExtractor::Cursor Cursor(0);
+
+        while (Cursor && !DE.eof(Cursor)) {
+          const uint64_t Input = DE.getAddress(Cursor);
+          const uint64_t Output = DE.getAddress(Cursor);
+          InsertCallback(Input, Output);
+        }
+
+        assert(Cursor && "Error reading address map section");
+        BC.deregisterSection(Section);
+      };
+
+  if (AddressMapSection) {
+    Parsed.Address2AddressMap.reserve(AddressMapSection->getOutputSize() /
+                                      EntrySize);
+    parseSection(*AddressMapSection, [&](uint64_t Input, uint64_t Output) {
+      if (!Parsed.Address2AddressMap.count(Input))
+        Parsed.Address2AddressMap.insert({Input, Output});
+    });
+  }
+
+  if (LabelMapSection) {
+    Parsed.Label2AddrMap.reserve(LabelMapSection->getOutputSize() / EntrySize);
+    parseSection(*LabelMapSection, [&](uint64_t Input, uint64_t Output) {
+      assert(!Parsed.Label2AddrMap.count(
+                 reinterpret_cast<const MCSymbol *>(Input)) &&
+             "Duplicate label entry detected.");
+      Parsed.Label2AddrMap.insert(
+          {reinterpret_cast<const MCSymbol *>(Input), Output});
+    });
   }
 
-  assert(Cursor && "Error reading address map section");
   return Parsed;
 }
 

bolt/lib/Core/BinaryContext.cpp

@@ -133,7 +133,7 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
     Expected<SubtargetFeatures> Features = File->getFeatures();
 
     if (auto E = Features.takeError())
-      return E;
+      return std::move(E);
 
     // We rely on relaxation for some transformations (e.g., promoting all calls
     // to PseudoCALL and then making JITLink relax them). Since the relax

bolt/lib/Core/BinaryFunction.cpp

@@ -4165,15 +4165,20 @@ void BinaryFunction::updateOutputValues(const BOLTLinker &Linker) {
       // Injected functions likely will fail lookup, as they have no
       // input range. Just assign the BB the output address of the
       // function.
-      auto MaybeBBAddress =
-          BC.getIOAddressMap().lookup(BB->getInputOffset() + getAddress());
+      auto MaybeBBAddress = BC.getIOAddressMap().lookup(BB->getLabel());
       const uint64_t BBAddress = MaybeBBAddress  ? *MaybeBBAddress
                                  : BB->isSplit() ? FF.getAddress()
                                                  : getOutputAddress();
       BB->setOutputStartAddress(BBAddress);
 
-      if (PrevBB)
+      if (PrevBB) {
+        assert(PrevBB->getOutputAddressRange().first <= BBAddress &&
+               "Bad output address for basic block.");
+        assert((PrevBB->getOutputAddressRange().first != BBAddress ||
+                !hasInstructions() || PrevBB->empty()) &&
+               "Bad output address for basic block.");
         PrevBB->setOutputEndAddress(BBAddress);
+      }
       PrevBB = BB;
     }
 

bolt/lib/Passes/HFSortPlus.cpp

@@ -567,7 +567,7 @@ class HFSortPlus {
     Into->Score = score(Into);
 
     // Remove chain From From the list of active chains
-    llvm::erase_value(HotChains, From);
+    llvm::erase(HotChains, From);
   }
 
 private:

bolt/lib/Passes/IndirectCallPromotion.cpp

@@ -591,7 +591,7 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector<Callsite> &Targets,
 
       NewTargets.push_back(Target);
       std::vector<uint64_t>({JTIndex}).swap(NewTargets.back().JTIndices);
-      llvm::erase_value(Target.JTIndices, JTIndex);
+      llvm::erase(Target.JTIndices, JTIndex);
 
       // Keep fixCFG counts sane if more indices use this same target later
       assert(IndicesPerTarget[Target.To.Sym] > 0 && "wrong map");

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -3196,9 +3196,6 @@ void RewriteInstance::preregisterSections() {
                               ROFlags);
   BC->registerOrUpdateSection(getNewSecPrefix() + ".rodata.cold",
                               ELF::SHT_PROGBITS, ROFlags);
-  BC->registerOrUpdateSection(AddressMap::SectionName, ELF::SHT_PROGBITS,
-                              ROFlags)
-      .setLinkOnly();
 }
 
 void RewriteInstance::emitAndLink() {
@@ -3669,11 +3666,8 @@ void RewriteInstance::mapAllocatableSections(
 }
 
 void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {
-  if (auto MapSection = BC->getUniqueSectionByName(AddressMap::SectionName)) {
-    auto Map = AddressMap::parse(MapSection->getOutputContents(), *BC);
-    BC->setIOAddressMap(std::move(Map));
-    BC->deregisterSection(*MapSection);
-  }
+  if (std::optional<AddressMap> Map = AddressMap::parse(*BC))
+    BC->setIOAddressMap(std::move(*Map));
 
   for (BinaryFunction *Function : BC->getAllBinaryFunctions())
     Function->updateOutputValues(Linker);

bolt/test/X86/jump-table-icp.test

@@ -12,6 +12,7 @@ RUN: (llvm-bolt %t.exe --data %t.fdata -o %t --relocs \
 RUN:   --reorder-blocks=cache --split-functions --split-all-cold \
 RUN:   --use-gnu-stack --dyno-stats --indirect-call-promotion=jump-tables \
 RUN:   --print-icp -v=0 \
+RUN:   --enable-bat --print-cache-metrics \
 RUN:   --icp-jt-remaining-percent-threshold=10 \
 RUN:   --icp-jt-total-percent-threshold=2 \
 RUN:   --indirect-call-promotion-topn=1 \