Merge remote-tracking branch 'intel/sycl' into cpu-range-reduction-performance

Author: steffenlarsen

.github/workflows/sycl_nightly.yml

@@ -19,6 +19,16 @@ jobs:
       build_configure_extra_args: ''
       lts_config: "ocl_gen9;ocl_x64"
 
+  ubuntu2004_opaque_pointers_build_test:
+    if: github.repository == 'intel/llvm'
+    uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    with:
+      build_cache_root: "/__w/"
+      build_cache_suffix: opaque_pointers
+      build_artifact_suffix: opaque_pointers
+      build_configure_extra_args: "--hip --cuda --enable-esimd-emulator --cmake-opt=-DDPCPP_ENABLE_OPAQUE_POINTERS=TRUE"
+      lts_config: "ocl_gen9;ocl_x64"
+
   windows_default:
     name: Windows
     if: github.repository == 'intel/llvm'

bolt/README.md

@@ -180,7 +180,7 @@ Once you have `perf.fdata` ready, you can use it for optimizations with
 BOLT. Assuming your environment is setup to include the right path, execute
 `llvm-bolt`:
 ```
-$ llvm-bolt <executable> -o <executable>.bolt -data=perf.fdata -reorder-blocks=ext-tsp -reorder-functions=hfsort -split-functions=2 -split-all-cold -split-eh -dyno-stats
+$ llvm-bolt <executable> -o <executable>.bolt -data=perf.fdata -reorder-blocks=ext-tsp -reorder-functions=hfsort -split-functions -split-all-cold -split-eh -dyno-stats
 ```
 
 If you do need an updated debug info, then add `-update-debug-sections` option

bolt/docs/OptimizingClang.md

@@ -64,7 +64,7 @@ Notice that we are passing `clang-7` to `perf2bolt` which is the real binary tha
 the generated profile:
 ```bash
 $ llvm-bolt $CPATH/clang-7 -o $CPATH/clang-7.bolt -b clang-7.yaml \
-    -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions=3 \
+    -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions \
     -split-all-cold -dyno-stats -icf=1 -use-gnu-stack
 ```
 The output will look similar to the one below:

bolt/include/bolt/Core/BinaryBasicBlock.h

@@ -15,6 +15,7 @@
 #ifndef BOLT_CORE_BINARY_BASIC_BLOCK_H
 #define BOLT_CORE_BINARY_BASIC_BLOCK_H
 
+#include "bolt/Core/FunctionLayout.h"
 #include "bolt/Core/MCPlus.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/StringRef.h"
@@ -634,14 +635,12 @@ class BinaryBasicBlock {
 
   /// Test if BB is a predecessor of this block.
   bool isPredecessor(const BinaryBasicBlock *BB) const {
-    auto Itr = std::find(Predecessors.begin(), Predecessors.end(), BB);
-    return Itr != Predecessors.end();
+    return llvm::is_contained(Predecessors, BB);
   }
 
   /// Test if BB is a successor of this block.
   bool isSuccessor(const BinaryBasicBlock *BB) const {
-    auto Itr = std::find(Successors.begin(), Successors.end(), BB);
-    return Itr != Successors.end();
+    return llvm::is_contained(Successors, BB);
   }
 
   /// Test if this BB has a valid execution count.
@@ -673,6 +672,10 @@ class BinaryBasicBlock {
 
   void markValid(const bool Valid) { IsValid = Valid; }
 
+  FragmentNum getFragmentNum() const {
+    return IsCold ? FragmentNum::cold() : FragmentNum::hot();
+  }
+
   bool isCold() const { return IsCold; }
 
   void setIsCold(const bool Flag) { IsCold = Flag; }

bolt/include/bolt/Core/BinaryContext.h

@@ -39,6 +39,7 @@
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/raw_ostream.h"
 #include <functional>
+#include <list>
 #include <map>
 #include <set>
 #include <shared_mutex>
@@ -199,7 +200,7 @@ class BinaryContext {
   uint32_t DuplicatedJumpTables{0x10000000};
 
   /// Function fragments to skip.
-  std::vector<BinaryFunction *> FragmentsToSkip;
+  std::unordered_set<BinaryFunction *> FragmentsToSkip;
 
   /// The runtime library.
   std::unique_ptr<RuntimeLibrary> RtLibrary;
@@ -235,6 +236,18 @@ class BinaryContext {
     MIB = std::move(TargetBuilder);
   }
 
+  /// Return function fragments to skip.
+  const std::unordered_set<BinaryFunction *> &getFragmentsToSkip() {
+    return FragmentsToSkip;
+  }
+
+  /// Add function fragment to skip
+  void addFragmentsToSkip(BinaryFunction *Function) {
+    FragmentsToSkip.insert(Function);
+  }
+
+  void clearFragmentsToSkip() { FragmentsToSkip.clear(); }
+
   /// Given DWOId returns CU if it exists in DWOCUs.
   Optional<DWARFUnit *> getDWOCU(uint64_t DWOId);
 
@@ -475,15 +488,15 @@ class BinaryContext {
   /// If \p NextJTAddress is different from zero, it is used as an upper
   /// bound for jump table memory layout.
   ///
-  /// Optionally, populate \p Offsets with jump table entries. The entries
+  /// Optionally, populate \p Address from jump table entries. The entries
   /// could be partially populated if the jump table detection fails.
   bool analyzeJumpTable(const uint64_t Address,
                         const JumpTable::JumpTableType Type, BinaryFunction &BF,
                         const uint64_t NextJTAddress = 0,
-                        JumpTable::OffsetsType *Offsets = nullptr);
+                        JumpTable::AddressesType *EntriesAsAddress = nullptr);
 
   /// After jump table locations are established, this function will populate
-  /// their OffsetEntries based on memory contents.
+  /// their EntriesAsAddress based on memory contents.
   void populateJumpTables();
 
   /// Returns a jump table ID and label pointing to the duplicated jump table.
@@ -498,12 +511,12 @@ class BinaryContext {
   /// to function \p BF.
   std::string generateJumpTableName(const BinaryFunction &BF, uint64_t Address);
 
-  /// Free memory used by jump table offsets
-  void clearJumpTableOffsets() {
+  /// Free memory used by JumpTable's EntriesAsAddress
+  void clearJumpTableTempData() {
     for (auto &JTI : JumpTables) {
       JumpTable &JT = *JTI.second;
-      JumpTable::OffsetsType Temp;
-      Temp.swap(JT.OffsetEntries);
+      JumpTable::AddressesType Temp;
+      Temp.swap(JT.EntriesAsAddress);
     }
   }
   /// Return true if the array of bytes represents a valid code padding.
@@ -641,6 +654,10 @@ class BinaryContext {
   /// special linux kernel sections
   std::unordered_map<uint64_t, std::vector<LKInstructionMarkerInfo>> LKMarkers;
 
+  /// List of external addresses in the code that are not a function start
+  /// and are referenced from BinaryFunction.
+  std::list<std::pair<BinaryFunction *, uint64_t>> InterproceduralReferences;
+
   /// PseudoProbe decoder
   MCPseudoProbeDecoder ProbeDecoder;
 
@@ -884,8 +901,23 @@ class BinaryContext {
   bool registerFragment(BinaryFunction &TargetFunction,
                         BinaryFunction &Function) const;
 
-  /// Resolve inter-procedural dependencies from \p Function.
-  void processInterproceduralReferences(BinaryFunction &Function);
+  /// Add unterprocedural reference for \p Function to \p Address
+  void addInterproceduralReference(BinaryFunction *Function, uint64_t Address) {
+    InterproceduralReferences.push_back({Function, Address});
+  }
+
+  /// Used to fix the target of linker-generated AArch64 adrp + add
+  /// sequence with no relocation info.
+  void addAdrpAddRelocAArch64(BinaryFunction &BF, MCInst &LoadLowBits,
+                              MCInst &LoadHiBits, uint64_t Target);
+
+  /// Return true if AARch64 veneer was successfully matched at a given
+  /// \p Address and register veneer binary function if \p MatchOnly
+  /// argument is false.
+  bool handleAArch64Veneer(uint64_t Address, bool MatchOnly = false);
+
+  /// Resolve inter-procedural dependencies from
+  void processInterproceduralReferences();
 
   /// Skip functions with all parent and child fragments transitively.
   void skipMarkedFragments();

bolt/include/bolt/Core/BinaryData.h

@@ -112,7 +112,7 @@ class BinaryData {
   bool nameStartsWith(StringRef Prefix) const;
 
   bool hasSymbol(const MCSymbol *Symbol) const {
-    return std::find(Symbols.begin(), Symbols.end(), Symbol) != Symbols.end();
+    return llvm::is_contained(Symbols, Symbol);
   }
 
   bool isAbsolute() const;