diff --git a/clang/cmake/modules/AddClang.cmake b/clang/cmake/modules/AddClang.cmake index 9bbbfc032b7df9..6a3e9dbf4c2bbe 100644 --- a/clang/cmake/modules/AddClang.cmake +++ b/clang/cmake/modules/AddClang.cmake @@ -160,7 +160,7 @@ macro(add_clang_tool name) get_target_export_arg(${name} Clang export_to_clangtargets) install(TARGETS ${name} ${export_to_clangtargets} - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR} COMPONENT ${name}) if(NOT LLVM_ENABLE_IDE) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp index 32b8f47ed6338e..aba3240c532bd5 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -217,7 +217,7 @@ namespace __sanitizer { unsigned struct_sockaddr_sz = sizeof(struct sockaddr); unsigned ucontext_t_sz(void *ctx) { -# if SANITIZER_GLIBC && SANITIZER_X64 +# if SANITIZER_GLIBC && SANITIZER_X64 && __GLIBC_PREREQ (2, 27) // See kernel arch/x86/kernel/fpu/signal.c for details. const auto *fpregs = static_cast(ctx)->uc_mcontext.fpregs; // The member names differ across header versions, but the actual layout diff --git a/compiler-rt/test/msan/Linux/signal_mcontext.cpp b/compiler-rt/test/msan/Linux/signal_mcontext.cpp index b49451fbb730b6..11ef74e7462bbe 100644 --- a/compiler-rt/test/msan/Linux/signal_mcontext.cpp +++ b/compiler-rt/test/msan/Linux/signal_mcontext.cpp @@ -10,7 +10,7 @@ void handler(int sig, siginfo_t *info, void *uctx) { __msan_check_mem_is_initialized(uctx, sizeof(ucontext_t)); -#if defined(__GLIBC__) && defined(__x86_64__) +#if defined(__GLIBC__) && defined(__x86_64__) && __GLIBC_PREREQ(2, 27) auto *mctx = &static_cast(uctx)->uc_mcontext; if (auto *fpregs = mctx->fpregs) { // The member names differ across header versions, but the actual layout diff --git a/libunwind/src/CompactUnwinder.hpp b/libunwind/src/CompactUnwinder.hpp index 0b2b5e111bfc26..ad0e042cb6ba11 100644 --- a/libunwind/src/CompactUnwinder.hpp +++ b/libunwind/src/CompactUnwinder.hpp @@ -310,6 +310,50 @@ int CompactUnwinder_x86_64::stepWithCompactEncodingRBPFrame( uint32_t savedRegistersLocations = EXTRACT_BITS(compactEncoding, UNWIND_X86_64_RBP_FRAME_REGISTERS); + // If we have not stored EBP yet + if (functionStart == registers.getIP()) { + uint64_t rsp = registers.getSP(); + // old esp is ebp less return address + registers.setSP(rsp+8); + // pop return address into eip + registers.setIP(addressSpace.get64(rsp)); + + return UNW_STEP_SUCCESS; + } else if (functionStart + 1 == registers.getIP()) { + uint64_t rsp = registers.getSP(); + // old esp is ebp less return address + registers.setSP(rsp + 16); + // pop return address into eip + registers.setIP(addressSpace.get64(rsp + 8)); + + return UNW_STEP_SUCCESS; + } + + // If we're about to return, we've already popped the base pointer + uint8_t b = addressSpace.get8(registers.getIP()); + + // This is a hack to detect VZEROUPPER but in between popq rbp and ret + // It's not pretty but it works + if (b == 0xC5) { + if ((b = addressSpace.get8(registers.getIP() + 1)) == 0xF8 && + (b = addressSpace.get8(registers.getIP() + 2)) == 0x77) + b = addressSpace.get8(registers.getIP() + 3); + else + goto skip_ret; + } + + if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) { + uint64_t rbp = registers.getSP(); + // old esp is ebp less return address + registers.setSP(rbp + 16); + // pop return address into eip + registers.setIP(addressSpace.get64(rbp + 8)); + + return UNW_STEP_SUCCESS; + } + + skip_ret: + uint64_t savedRegisters = registers.getRBP() - 8 * savedRegistersOffset; for (int i = 0; i < 5; ++i) { switch (savedRegistersLocations & 0x7) { @@ -430,6 +474,118 @@ int CompactUnwinder_x86_64::stepWithCompactEncodingFrameless( } } } + + // Note that the order of these registers is so that + // registersSaved[0] is the one that will be pushed onto the stack last. + // Thus, if we want to walk this from the top, we need to go in reverse. + assert(regCount <= 6); + + // check whether we are still in the prologue + uint64_t curAddr = functionStart; + if (regCount > 0) { + for (int8_t i = (int8_t)(regCount) - 1; i >= 0; --i) { + if (registers.getIP() == curAddr) { + // None of the registers have been modified yet, so we don't need to reload them + framelessUnwind(addressSpace, registers.getSP() + 8 * (regCount - (uint64_t)(i + 1)), registers); + return UNW_STEP_SUCCESS; + } else { + assert(curAddr < registers.getIP()); + } + + + // pushq %rbp and pushq %rbx is 1 byte. Everything else 2 + if ((UNWIND_X86_64_REG_RBP == registersSaved[i]) || + (UNWIND_X86_64_REG_RBX == registersSaved[i])) + curAddr += 1; + else + curAddr += 2; + } + } + if (registers.getIP() == curAddr) { + // None of the registers have been modified yet, so we don't need to reload them + framelessUnwind(addressSpace, registers.getSP() + 8*regCount, registers); + return UNW_STEP_SUCCESS; + } else { + assert(curAddr < registers.getIP()); + } + + + // And now for the epilogue + { + uint8_t i = 0; + uint64_t p = registers.getIP(); + uint8_t b = 0; + + while (true) { + b = addressSpace.get8(p++); + // This is a hack to detect VZEROUPPER but in between the popq's and ret + // It's not pretty but it works + if (b == 0xC5) { + if ((b = addressSpace.get8(p++)) == 0xF8 && (b = addressSpace.get8(p++)) == 0x77) + b = addressSpace.get8(p++); + else + break; + } + // popq %rbx popq %rbp + if (b == 0x5B || b == 0x5D) { + i++; + } else if (b == 0x41) { + b = addressSpace.get8(p++); + if (b == 0x5C || b == 0x5D || b == 0x5E || b == 0x5F) + i++; + else + break; + } else if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) { + // i pop's haven't happened yet + uint64_t savedRegisters = registers.getSP() + 8 * i; + if (regCount > 0) { + for (int8_t j = (int8_t)(regCount) - 1; j >= (int8_t)(regCount) - i; --j) { + uint64_t addr = savedRegisters - 8 * (regCount - (uint64_t)(j)); + switch (registersSaved[j]) { + case UNWIND_X86_64_REG_RBX: + registers.setRBX(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R12: + registers.setR12(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R13: + registers.setR13(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R14: + registers.setR14(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R15: + registers.setR15(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_RBP: + registers.setRBP(addressSpace.get64(addr)); + break; + default: + _LIBUNWIND_DEBUG_LOG("bad register for frameless, encoding=%08X for " + "function starting at 0x%llX", + encoding, functionStart); + _LIBUNWIND_ABORT("invalid compact unwind encoding"); + } + } + } + framelessUnwind(addressSpace, savedRegisters, registers); + return UNW_STEP_SUCCESS; + } else { + break; + } + } + } + + /* + 0x10fe2733a: 5b popq %rbx + 0x10fe2733b: 41 5c popq %r12 + 0x10fe2733d: 41 5d popq %r13 + 0x10fe2733f: 41 5e popq %r14 + 0x10fe27341: 41 5f popq %r15 + 0x10fe27343: 5d popq %rbp + */ + + uint64_t savedRegisters = registers.getSP() + stackSize - 8 - 8 * regCount; for (uint32_t i = 0; i < regCount; ++i) { switch (registersSaved[i]) { diff --git a/lld/cmake/modules/AddLLD.cmake b/lld/cmake/modules/AddLLD.cmake index dd2898ce623647..5e176976b60300 100644 --- a/lld/cmake/modules/AddLLD.cmake +++ b/lld/cmake/modules/AddLLD.cmake @@ -20,7 +20,7 @@ macro(add_lld_library name) ${export_to_lldtargets} LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX} ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR}) if (${ARG_SHARED} AND NOT CMAKE_CONFIGURATION_TYPES) add_llvm_install_targets(install-${name} @@ -47,7 +47,7 @@ macro(add_lld_tool name) get_target_export_arg(${name} LLD export_to_lldtargets) install(TARGETS ${name} ${export_to_lldtargets} - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR} COMPONENT ${name}) if(NOT CMAKE_CONFIGURATION_TYPES) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h b/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h new file mode 100644 index 00000000000000..d748d4b0fa5927 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h @@ -0,0 +1,35 @@ +//===--------- DWARFRecordSectionSplitter.h - JITLink -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H +#define LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// A LinkGraph pass that splits blocks in a section that follows the DWARF +/// Record format into sub-blocks where each header gets its own block. +/// When splitting EHFrames, DWARFRecordSectionSplitter should not be run +/// without EHFrameEdgeFixer, which is responsible for adding FDE-to-CIE edges. +class DWARFRecordSectionSplitter { +public: + DWARFRecordSectionSplitter(StringRef SectionName); + Error operator()(LinkGraph &G); + +private: + Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); + + StringRef SectionName; +}; + +} // namespace jitlink +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h index aee14c0d1fe5e8..6f2ff012697ddb 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -18,30 +18,6 @@ namespace llvm { namespace jitlink { -namespace MachO_arm64_Edges { - -enum MachOARM64RelocationKind : Edge::Kind { - Branch26 = Edge::FirstRelocation, - Pointer32, - Pointer64, - Pointer64Anon, - Page21, - PageOffset12, - GOTPage21, - GOTPageOffset12, - TLVPage21, - TLVPageOffset12, - PointerToGOT, - PairedAddend, - LDRLiteral19, - Delta32, - Delta64, - NegDelta32, - NegDelta64, -}; - -} // namespace MachO_arm64_Edges - /// Create a LinkGraph from a MachO/arm64 relocatable object. /// /// Note: The graph does not take ownership of the underlying buffer, nor copy @@ -62,9 +38,6 @@ createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer); void link_MachO_arm64(std::unique_ptr G, std::unique_ptr Ctx); -/// Return the string name of the given MachO arm64 edge kind. -const char *getMachOARM64RelocationKindName(Edge::Kind R); - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 994ce783b05863..42a01c25de146b 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -13,24 +13,352 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H #define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H +#include "TableManager.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" namespace llvm { namespace jitlink { namespace aarch64 { -/// Represets aarch64 fixups enum EdgeKind_aarch64 : Edge::Kind { - - /// Set a CALL immediate field to bits [27:2] of X = Target - Fixup + Addend - R_AARCH64_CALL26 = Edge::FirstRelocation, - + Branch26 = Edge::FirstRelocation, + Pointer32, + Pointer64, + Pointer64Anon, + Page21, + PageOffset12, + MoveWide16, + GOTPage21, + GOTPageOffset12, + TLVPage21, + TLVPageOffset12, + PointerToGOT, + PairedAddend, + LDRLiteral19, + Delta32, + Delta64, + NegDelta32, + NegDelta64, }; /// Returns a string name for the given aarch64 edge. For debugging purposes /// only const char *getEdgeKindName(Edge::Kind K); +// Returns whether the Instr is LD/ST (imm12) +inline bool isLoadStoreImm12(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + return (Instr & LoadStoreImm12Mask) == 0x39000000; +} + +// Returns the amount the address operand of LD/ST (imm12) +// should be shifted right by. +// +// The shift value varies by the data size of LD/ST instruction. +// For instance, LDH instructoin needs the address to be shifted +// right by 1. +inline unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t Vec128Mask = 0x04800000; + + if (isLoadStoreImm12(Instr)) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; +} + +// Returns whether the Instr is MOVK/MOVZ (imm16) with a zero immediate field +inline bool isMoveWideImm16(uint32_t Instr) { + constexpr uint32_t MoveWideImm16Mask = 0x5f9fffe0; + return (Instr & MoveWideImm16Mask) == 0x52800000; +} + +// Returns the amount the address operand of MOVK/MOVZ (imm16) +// should be shifted right by. +// +// The shift value is specfied in the assembly as LSL #. +inline unsigned getMoveWide16Shift(uint32_t Instr) { + if (isMoveWideImm16(Instr)) { + uint32_t ImplicitShift = (Instr >> 21) & 0b11; + return ImplicitShift << 4; + } + + return 0; +} + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast(Value) & 0x3) + return make_error("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (!isInt<33>(PageDelta)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case MoveWide16: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue(); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(isMoveWideImm16(RawInstr) && + "RawInstr isn't a MOVK/MOVZ instruction"); + + unsigned ImmShift = getMoveWide16Shift(RawInstr); + uint32_t Imm = (TargetOffset >> ImmShift) & 0xffff; + uint32_t FixedInstr = RawInstr | (Imm << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); + } + + return Error::success(); +} + +/// AArch64 null pointer content. +extern const uint8_t NullGOTEntryContent[8]; + +/// AArch64 PLT stub content. +extern const uint8_t StubContent[8]; + +/// Global Offset Table Builder. +class GOTTableManager : public TableManager { +public: + static StringRef getSectionName() { return "$__GOT"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind KindToSet = Edge::Invalid; + const char *BlockWorkingMem = B->getContent().data(); + const char *FixupPtr = BlockWorkingMem + E.getOffset(); + + switch (E.getKind()) { + case aarch64::GOTPage21: + case aarch64::TLVPage21: { + KindToSet = aarch64::Page21; + break; + } + case aarch64::GOTPageOffset12: + case aarch64::TLVPageOffset12: { + KindToSet = aarch64::PageOffset12; + uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr; + assert(E.getAddend() == 0 && + "GOTPageOffset12/TLVPageOffset12 with non-zero addend"); + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + break; + } + case aarch64::PointerToGOT: { + KindToSet = aarch64::Delta64; + break; + } + default: + return false; + } + assert(KindToSet != Edge::Invalid && + "Fell through switch, but no new kind to set"); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(KindToSet); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(G), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); + GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + +private: + Section &getGOTSection(LinkGraph &G) { + if (!GOTSection) + GOTSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *GOTSection; + } + + ArrayRef getGOTEntryBlockContent() { + return {reinterpret_cast(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; + } + + Section *GOTSection = nullptr; +}; + +/// Procedure Linkage Table Builder. +class PLTTableManager : public TableManager { +public: + PLTTableManager(GOTTableManager &GOT) : GOT(GOT) {} + + static StringRef getSectionName() { return "$__STUBS"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined()) { + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + return false; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &StubContentBlock = G.createContentBlock( + getStubsSection(G), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = GOT.getEntryForTarget(G, Target); + StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); + } + +public: + Section &getStubsSection(LinkGraph &G) { + if (!StubsSection) + StubsSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *StubsSection; + } + + ArrayRef getStubBlockContent() { + return {reinterpret_cast(StubContent), sizeof(StubContent)}; + } + + GOTTableManager &GOT; + Section *StubsSection = nullptr; +}; + } // namespace aarch64 } // namespace jitlink } // namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h index 4a4e8d15be6640..9a2bc9b09350eb 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -447,11 +447,10 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, break; } - default: { - // If you hit this you should check that *constructor and other non-fixup - // edges have been removed prior to applying fixups. - llvm_unreachable("Graph contains edge kind with no fixup expression"); - } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); } return Error::success(); diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td index b01fa10763b83e..6e5b9cbd9a4499 100644 --- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td +++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td @@ -1459,7 +1459,11 @@ def int_ppc_tsuspend : GCCBuiltin<"__builtin_tsuspend">, def int_ppc_ttest : GCCBuiltin<"__builtin_ttest">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_cfence : Intrinsic<[], [llvm_anyint_ty], []>; +// We currently use llvm.ppc.cfence in the context of atomic load which +// in LLVM IR requires its type to be one of integer, pointer and +// float point type. So llvm_any_ty here refers to type mentioned above. +// Backend is supposed to lower these types to appropriate MVTs. +def int_ppc_cfence : Intrinsic<[], [llvm_any_ty], []>; // PowerPC set FPSCR Intrinsic Definitions. def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">, diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index c888adeafca5d1..328412821f20de 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -5205,18 +5205,29 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, WeakTrackingVH SunkAddrVH = SunkAddrs[Addr]; Value * SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr; + Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); if (SunkAddr) { LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for " << *MemoryInst << "\n"); - if (SunkAddr->getType() != Addr->getType()) - SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + if (SunkAddr->getType() != Addr->getType()) { + // Even though we only considered no-op addrspacecasts, + // semantically-meaningful conversions may still be present due to + // ptrtoint/inttoptr sequences. + if (SunkAddr->getType()->getPointerAddressSpace() != + Addr->getType()->getPointerAddressSpace() && + !DL->isNonIntegralPointerType(Addr->getType())) { + SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr"); + SunkAddr = + Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr"); + } else + SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + } } else if (AddrSinkUsingGEPs || (!AddrSinkUsingGEPs.getNumOccurrences() && SubtargetInfo->addrSinkUsingGEPs())) { // By default, we use the GEP-based method when AA is used later. This // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities. LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for " << *MemoryInst << "\n"); - Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); Value *ResultPtr = nullptr, *ResultIndex = nullptr; // First, find the pointer. @@ -5345,8 +5356,19 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, : Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); } - if (SunkAddr->getType() != Addr->getType()) - SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + if (SunkAddr->getType() != Addr->getType()) { + // Even though we only considered no-op addrspacecasts, + // semantically-meaningful conversions may still be present due to + // ptrtoint/inttoptr sequences. + if (SunkAddr->getType()->getPointerAddressSpace() != + Addr->getType()->getPointerAddressSpace() && + !DL->isNonIntegralPointerType(Addr->getType())) { + SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr"); + SunkAddr = + Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr"); + } else + SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + } } } else { // We'd require a ptrtoint/inttoptr down the line, which we can't do for diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt index f87b3b895137e2..5a5bdae3a1bd8a 100644 --- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_component_library(LLVMJITLink + DWARFRecordSectionSplitter.cpp EHFrameSupport.cpp JITLink.cpp JITLinkGeneric.cpp diff --git a/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp b/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp new file mode 100644 index 00000000000000..0fc366bf505f16 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp @@ -0,0 +1,117 @@ +//===-------- JITLink_DWARFRecordSectionSplitter.cpp - JITLink-------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" +#include "llvm/Support/BinaryStreamReader.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +DWARFRecordSectionSplitter::DWARFRecordSectionSplitter(StringRef SectionName) + : SectionName(SectionName) {} + +Error DWARFRecordSectionSplitter::operator()(LinkGraph &G) { + auto *Section = G.findSectionByName(SectionName); + + if (!Section) { + LLVM_DEBUG({ + dbgs() << "DWARFRecordSectionSplitter: No " << SectionName + << " section. Nothing to do\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "DWARFRecordSectionSplitter: Processing " << SectionName + << "...\n"; + }); + + DenseMap Caches; + + { + // Pre-build the split caches. + for (auto *B : Section->blocks()) + Caches[B] = LinkGraph::SplitBlockCache::value_type(); + for (auto *Sym : Section->symbols()) + Caches[&Sym->getBlock()]->push_back(Sym); + for (auto *B : Section->blocks()) + llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + + // Iterate over blocks (we do this by iterating over Caches entries rather + // than Section->blocks() as we will be inserting new blocks along the way, + // which would invalidate iterators in the latter sequence. + for (auto &KV : Caches) { + auto &B = *KV.first; + auto &BCache = KV.second; + if (auto Err = processBlock(G, B, BCache)) + return Err; + } + + return Error::success(); +} + +Error DWARFRecordSectionSplitter::processBlock( + LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache) { + LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); + + // Section should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error("Unexpected zero-fill block in " + + SectionName + " section"); + + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + BinaryStreamReader BlockReader( + StringRef(B.getContent().data(), B.getContent().size()), + G.getEndianness()); + + while (true) { + uint64_t RecordStartOffset = BlockReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress()) << "\n"; + }); + + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + if (Length != 0xffffffff) { + if (auto Err = BlockReader.skip(Length)) + return Err; + } else { + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + if (auto Err = BlockReader.skip(ExtendedLength)) + return Err; + } + + // If this was the last block then there's nothing to split + if (BlockReader.empty()) { + LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); + return Error::success(); + } + + uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; + auto &NewBlock = G.splitBlock(B, BlockSize); + (void)NewBlock; + LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); + } +} + +} // namespace jitlink +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 2ae193595fc021..25a18d45e27b4d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -10,6 +10,7 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/Support/DynamicLibrary.h" @@ -18,109 +19,13 @@ namespace llvm { namespace jitlink { -EHFrameSplitter::EHFrameSplitter(StringRef EHFrameSectionName) - : EHFrameSectionName(EHFrameSectionName) {} - -Error EHFrameSplitter::operator()(LinkGraph &G) { - auto *EHFrame = G.findSectionByName(EHFrameSectionName); - - if (!EHFrame) { - LLVM_DEBUG({ - dbgs() << "EHFrameSplitter: No " << EHFrameSectionName - << " section. Nothing to do\n"; - }); - return Error::success(); - } - - LLVM_DEBUG({ - dbgs() << "EHFrameSplitter: Processing " << EHFrameSectionName << "...\n"; - }); - - DenseMap Caches; - - { - // Pre-build the split caches. - for (auto *B : EHFrame->blocks()) - Caches[B] = LinkGraph::SplitBlockCache::value_type(); - for (auto *Sym : EHFrame->symbols()) - Caches[&Sym->getBlock()]->push_back(Sym); - for (auto *B : EHFrame->blocks()) - llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { - return LHS->getOffset() > RHS->getOffset(); - }); - } - - // Iterate over blocks (we do this by iterating over Caches entries rather - // than EHFrame->blocks() as we will be inserting new blocks along the way, - // which would invalidate iterators in the latter sequence. - for (auto &KV : Caches) { - auto &B = *KV.first; - auto &BCache = KV.second; - if (auto Err = processBlock(G, B, BCache)) - return Err; - } - - return Error::success(); -} - -Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, - LinkGraph::SplitBlockCache &Cache) { - LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); - - // eh-frame should not contain zero-fill blocks. - if (B.isZeroFill()) - return make_error("Unexpected zero-fill block in " + - EHFrameSectionName + " section"); - - if (B.getSize() == 0) { - LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); - return Error::success(); - } - - BinaryStreamReader BlockReader( - StringRef(B.getContent().data(), B.getContent().size()), - G.getEndianness()); - - while (true) { - uint64_t RecordStartOffset = BlockReader.getOffset(); - - LLVM_DEBUG({ - dbgs() << " Processing CFI record at " - << formatv("{0:x16}", B.getAddress()) << "\n"; - }); - - uint32_t Length; - if (auto Err = BlockReader.readInteger(Length)) - return Err; - if (Length != 0xffffffff) { - if (auto Err = BlockReader.skip(Length)) - return Err; - } else { - uint64_t ExtendedLength; - if (auto Err = BlockReader.readInteger(ExtendedLength)) - return Err; - if (auto Err = BlockReader.skip(ExtendedLength)) - return Err; - } - - // If this was the last block then there's nothing to split - if (BlockReader.empty()) { - LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); - return Error::success(); - } - - uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; - auto &NewBlock = G.splitBlock(B, BlockSize); - (void)NewBlock; - LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); - } -} - EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, - unsigned PointerSize, Edge::Kind Delta64, - Edge::Kind Delta32, Edge::Kind NegDelta32) + unsigned PointerSize, Edge::Kind Pointer32, + Edge::Kind Pointer64, Edge::Kind Delta32, + Edge::Kind Delta64, Edge::Kind NegDelta32) : EHFrameSectionName(EHFrameSectionName), PointerSize(PointerSize), - Delta64(Delta64), Delta32(Delta32), NegDelta32(NegDelta32) {} + Pointer32(Pointer32), Pointer64(Pointer64), Delta32(Delta32), + Delta64(Delta64), NegDelta32(NegDelta32) {} Error EHFrameEdgeFixer::operator()(LinkGraph &G) { auto *EHFrame = G.findSectionByName(EHFrameSectionName); @@ -147,7 +52,16 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) { // Build a map of all blocks and symbols in the text sections. We will use // these for finding / building edge targets when processing FDEs. for (auto &Sec : G.sections()) { - PC.AddrToSyms.addSymbols(Sec.symbols()); + // Just record the most-canonical symbol (for eh-frame purposes) at each + // address. + for (auto *Sym : Sec.symbols()) { + auto &CurSym = PC.AddrToSym[Sym->getAddress()]; + if (!CurSym || (std::make_tuple(Sym->getLinkage(), Sym->getScope(), + !Sym->hasName(), Sym->getName()) < + std::make_tuple(CurSym->getLinkage(), CurSym->getScope(), + !CurSym->hasName(), CurSym->getName()))) + CurSym = Sym; + } if (auto Err = PC.AddrToBlock.addBlocks(Sec.blocks(), BlockAddressMap::includeNonNull)) return Err; @@ -172,10 +86,7 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) { Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { - LLVM_DEBUG({ - dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) - << "\n"; - }); + LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); // eh-frame should not contain zero-fill blocks. if (B.isZeroFill()) @@ -209,7 +120,7 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { LLVM_DEBUG({ dbgs() << " Processing CFI record at " - << formatv("{0:x16}", B.getAddress() + RecordStartOffset) << "\n"; + << (B.getAddress() + RecordStartOffset) << "\n"; }); // Get the record length. @@ -244,7 +155,7 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { if (CIEDelta == 0) { if (auto Err = processCIE(PC, B, RecordStartOffset, CIEDeltaFieldOffset + RecordRemaining, - CIEDeltaFieldOffset)) + CIEDeltaFieldOffset, BlockEdges)) return Err; } else { if (auto Err = processFDE(PC, B, RecordStartOffset, @@ -263,7 +174,8 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, - size_t CIEDeltaFieldOffset) { + size_t CIEDeltaFieldOffset, + const BlockEdgeMap &BlockEdges) { LLVM_DEBUG(dbgs() << " Record is CIE\n"); @@ -301,10 +213,6 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, uint64_t CodeAlignmentFactor = 0; if (auto Err = RecordReader.readULEB128(CodeAlignmentFactor)) return Err; - if (CodeAlignmentFactor != 1) - return make_error("Unsupported CIE code alignment factor " + - Twine(CodeAlignmentFactor) + - " (expected 1)"); } // Read and validate the data alignment factor. @@ -312,76 +220,65 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, int64_t DataAlignmentFactor = 0; if (auto Err = RecordReader.readSLEB128(DataAlignmentFactor)) return Err; - if (DataAlignmentFactor != -8) - return make_error("Unsupported CIE data alignment factor " + - Twine(DataAlignmentFactor) + - " (expected -8)"); } // Skip the return address register field. if (auto Err = RecordReader.skip(1)) return Err; - uint64_t AugmentationDataLength = 0; - if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) - return Err; + if (AugInfo->AugmentationDataPresent) { - uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); + CIEInfo.AugmentationDataPresent = true; - uint8_t *NextField = &AugInfo->Fields[0]; - while (uint8_t Field = *NextField++) { - switch (Field) { - case 'L': { - CIEInfo.FDEsHaveLSDAField = true; - uint8_t LSDAPointerEncoding; - if (auto Err = RecordReader.readInteger(LSDAPointerEncoding)) - return Err; - if (!isSupportedPointerEncoding(LSDAPointerEncoding)) - return make_error( - "Unsupported LSDA pointer encoding " + - formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CIESymbol.getAddress())); - CIEInfo.LSDAPointerEncoding = LSDAPointerEncoding; - break; - } - case 'P': { - uint8_t PersonalityPointerEncoding = 0; - if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding)) - return Err; - if (PersonalityPointerEncoding != - (dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | - dwarf::DW_EH_PE_sdata4)) - return make_error( - "Unspported personality pointer " - "encoding " + - formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CIESymbol.getAddress())); - uint32_t PersonalityPointerAddress; - if (auto Err = RecordReader.readInteger(PersonalityPointerAddress)) - return Err; - break; - } - case 'R': { - uint8_t FDEPointerEncoding; - if (auto Err = RecordReader.readInteger(FDEPointerEncoding)) - return Err; - if (!isSupportedPointerEncoding(FDEPointerEncoding)) - return make_error( - "Unsupported FDE pointer encoding " + - formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CIESymbol.getAddress())); - CIEInfo.FDEPointerEncoding = FDEPointerEncoding; - break; - } - default: - llvm_unreachable("Invalid augmentation string field"); + uint64_t AugmentationDataLength = 0; + if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': + CIEInfo.LSDAPresent = true; + if (auto PE = readPointerEncoding(RecordReader, B, "LSDA")) + CIEInfo.LSDAEncoding = *PE; + else + return PE.takeError(); + break; + case 'P': { + auto PersonalityPointerEncoding = + readPointerEncoding(RecordReader, B, "personality"); + if (!PersonalityPointerEncoding) + return PersonalityPointerEncoding.takeError(); + if (auto Err = + getOrCreateEncodedPointerEdge( + PC, BlockEdges, *PersonalityPointerEncoding, RecordReader, + B, RecordOffset + RecordReader.getOffset(), "personality") + .takeError()) + return Err; + break; + } + case 'R': + if (auto PE = readPointerEncoding(RecordReader, B, "address")) { + CIEInfo.AddressEncoding = *PE; + if (CIEInfo.AddressEncoding == dwarf::DW_EH_PE_omit) + return make_error( + "Invalid address encoding DW_EH_PE_omit in CIE at " + + formatv("{0:x}", (B.getAddress() + RecordOffset).getValue())); + } else + return PE.takeError(); + break; + default: + llvm_unreachable("Invalid augmentation string field"); + } } - } - if (RecordReader.getOffset() - AugmentationDataStartOffset > - AugmentationDataLength) - return make_error("Read past the end of the augmentation " - "data while parsing fields"); + if (RecordReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error("Read past the end of the augmentation " + "data while parsing fields"); + } assert(!PC.CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); @@ -394,7 +291,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset, uint32_t CIEDelta, - BlockEdgeMap &BlockEdges) { + const BlockEdgeMap &BlockEdges) { LLVM_DEBUG(dbgs() << " Record is FDE\n"); orc::ExecutorAddr RecordAddress = B.getAddress() + RecordOffset; @@ -422,8 +319,8 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, LLVM_DEBUG({ dbgs() << " Adding edge at " - << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) - << " to CIE at: " << formatv("{0:x16}", CIEAddress) << "\n"; + << (RecordAddress + CIEDeltaFieldOffset) + << " to CIE at: " << CIEAddress << "\n"; }); if (auto CIEInfoOrErr = PC.findCIEInfo(CIEAddress)) CIEInfo = *CIEInfoOrErr; @@ -435,8 +332,8 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, } else { LLVM_DEBUG({ dbgs() << " Already has edge at " - << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) - << " to CIE at " << formatv("{0:x16}", CIEAddress) << "\n"; + << (RecordAddress + CIEDeltaFieldOffset) << " to CIE at " + << CIEAddress << "\n"; }); auto &EI = CIEEdgeItr->second; if (EI.Addend) @@ -451,107 +348,41 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, } } - { - // Process the PC-Begin field. - Block *PCBeginBlock = nullptr; - orc::ExecutorAddrDiff PCBeginFieldOffset = RecordReader.getOffset(); - auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); - if (PCEdgeItr == BlockEdges.end()) { - auto PCBeginPtrInfo = - readEncodedPointer(CIEInfo->FDEPointerEncoding, - RecordAddress + PCBeginFieldOffset, RecordReader); - if (!PCBeginPtrInfo) - return PCBeginPtrInfo.takeError(); - orc::ExecutorAddr PCBegin = PCBeginPtrInfo->first; - Edge::Kind PCBeginEdgeKind = PCBeginPtrInfo->second; - LLVM_DEBUG({ - dbgs() << " Adding edge at " - << (RecordAddress + PCBeginFieldOffset) << " to PC at " - << formatv("{0:x16}", PCBegin) << "\n"; - }); - auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); - if (!PCBeginSym) - return PCBeginSym.takeError(); - B.addEdge(PCBeginEdgeKind, RecordOffset + PCBeginFieldOffset, *PCBeginSym, - 0); - PCBeginBlock = &PCBeginSym->getBlock(); - } else { - auto &EI = PCEdgeItr->second; - LLVM_DEBUG({ - dbgs() << " Already has edge at " - << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) - << " to PC at " << formatv("{0:x16}", EI.Target->getAddress()); - if (EI.Addend) - dbgs() << " + " << formatv("{0:x16}", EI.Addend); - dbgs() << "\n"; - }); - - // Make sure the existing edge points at a defined block. - if (!EI.Target->isDefined()) { - auto EdgeAddr = RecordAddress + PCBeginFieldOffset; - return make_error("FDE edge at " + - formatv("{0:x16}", EdgeAddr) + - " points at external block"); - } - PCBeginBlock = &EI.Target->getBlock(); - if (auto Err = RecordReader.skip( - getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) - return Err; - } - + // Process the PC-Begin field. + LLVM_DEBUG({ + dbgs() << " Processing PC-begin at " + << (RecordAddress + RecordReader.getOffset()) << "\n"; + }); + if (auto PCBegin = getOrCreateEncodedPointerEdge( + PC, BlockEdges, CIEInfo->AddressEncoding, RecordReader, B, + RecordReader.getOffset(), "PC begin")) { + assert(*PCBegin && "PC-begin symbol not set"); // Add a keep-alive edge from the FDE target to the FDE to ensure that the // FDE is kept alive if its target is. - assert(PCBeginBlock && "PC-begin block not recorded"); LLVM_DEBUG({ dbgs() << " Adding keep-alive edge from target at " - << formatv("{0:x16}", PCBeginBlock->getAddress()) << " to FDE at " - << formatv("{0:x16}", RecordAddress) << "\n"; + << (*PCBegin)->getBlock().getAddress() << " to FDE at " + << RecordAddress << "\n"; }); - PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0); - } + (*PCBegin)->getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + } else + return PCBegin.takeError(); // Skip over the PC range size field. - if (auto Err = RecordReader.skip( - getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) + if (auto Err = skipEncodedPointer(CIEInfo->AddressEncoding, RecordReader)) return Err; - if (CIEInfo->FDEsHaveLSDAField) { + if (CIEInfo->AugmentationDataPresent) { uint64_t AugmentationDataSize; if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) return Err; - orc::ExecutorAddrDiff LSDAFieldOffset = RecordReader.getOffset(); - auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); - if (LSDAEdgeItr == BlockEdges.end()) { - auto LSDAPointerInfo = - readEncodedPointer(CIEInfo->LSDAPointerEncoding, - RecordAddress + LSDAFieldOffset, RecordReader); - if (!LSDAPointerInfo) - return LSDAPointerInfo.takeError(); - orc::ExecutorAddr LSDA = LSDAPointerInfo->first; - Edge::Kind LSDAEdgeKind = LSDAPointerInfo->second; - auto LSDASym = getOrCreateSymbol(PC, LSDA); - if (!LSDASym) - return LSDASym.takeError(); - LLVM_DEBUG({ - dbgs() << " Adding edge at " - << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) - << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n"; - }); - B.addEdge(LSDAEdgeKind, RecordOffset + LSDAFieldOffset, *LSDASym, 0); - } else { - LLVM_DEBUG({ - auto &EI = LSDAEdgeItr->second; - dbgs() << " Already has edge at " - << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) - << " to LSDA at " << formatv("{0:x16}", EI.Target->getAddress()); - if (EI.Addend) - dbgs() << " + " << formatv("{0:x16}", EI.Addend); - dbgs() << "\n"; - }); - if (auto Err = RecordReader.skip(AugmentationDataSize)) + if (CIEInfo->LSDAPresent) + if (auto Err = getOrCreateEncodedPointerEdge( + PC, BlockEdges, CIEInfo->LSDAEncoding, RecordReader, B, + RecordReader.getOffset(), "LSDA") + .takeError()) return Err; - } } else { LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); } @@ -600,129 +431,163 @@ EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) { return std::move(AugInfo); } -bool EHFrameEdgeFixer::isSupportedPointerEncoding(uint8_t PointerEncoding) { +Expected EHFrameEdgeFixer::readPointerEncoding(BinaryStreamReader &R, + Block &InBlock, + const char *FieldName) { using namespace dwarf; - // We only support PC-rel for now. - if ((PointerEncoding & 0x70) != DW_EH_PE_pcrel) - return false; - - // readEncodedPointer does not handle indirect. - if (PointerEncoding & DW_EH_PE_indirect) - return false; + uint8_t PointerEncoding; + if (auto Err = R.readInteger(PointerEncoding)) + return std::move(Err); - // Supported datatypes. + bool Supported = true; switch (PointerEncoding & 0xf) { - case DW_EH_PE_absptr: - case DW_EH_PE_udata4: - case DW_EH_PE_udata8: - case DW_EH_PE_sdata4: - case DW_EH_PE_sdata8: - return true; + case DW_EH_PE_uleb128: + case DW_EH_PE_udata2: + case DW_EH_PE_sleb128: + case DW_EH_PE_sdata2: + Supported = false; + break; + } + if (Supported) { + switch (PointerEncoding & 0x70) { + case DW_EH_PE_textrel: + case DW_EH_PE_datarel: + case DW_EH_PE_funcrel: + case DW_EH_PE_aligned: + Supported = false; + break; + } } - return false; + if (Supported) + return PointerEncoding; + + return make_error("Unsupported pointer encoding " + + formatv("{0:x2}", PointerEncoding) + " for " + + FieldName + "in CFI record at " + + formatv("{0:x16}", InBlock.getAddress())); } -unsigned EHFrameEdgeFixer::getPointerEncodingDataSize(uint8_t PointerEncoding) { +Error EHFrameEdgeFixer::skipEncodedPointer(uint8_t PointerEncoding, + BinaryStreamReader &RecordReader) { using namespace dwarf; - assert(isSupportedPointerEncoding(PointerEncoding) && - "Unsupported pointer encoding"); + // Switch absptr to corresponding udata encoding. + if ((PointerEncoding & 0xf) == DW_EH_PE_absptr) + PointerEncoding |= (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + switch (PointerEncoding & 0xf) { - case DW_EH_PE_absptr: - return PointerSize; case DW_EH_PE_udata4: case DW_EH_PE_sdata4: - return 4; + if (auto Err = RecordReader.skip(4)) + return Err; + break; case DW_EH_PE_udata8: case DW_EH_PE_sdata8: - return 8; + if (auto Err = RecordReader.skip(8)) + return Err; + break; default: - llvm_unreachable("Unsupported encoding"); + llvm_unreachable("Unrecognized encoding"); } + return Error::success(); } -Expected> -EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, - orc::ExecutorAddr PointerFieldAddress, - BinaryStreamReader &RecordReader) { - assert(isSupportedPointerEncoding(PointerEncoding) && - "Unsupported pointer encoding"); - +Expected EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( + ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding, + BinaryStreamReader &RecordReader, Block &BlockToFix, + size_t PointerFieldOffset, const char *FieldName) { using namespace dwarf; - // Isolate data type, remap absptr to udata4 or udata8. This relies on us - // having verified that the graph uses 32-bit or 64-bit pointers only at the - // start of this pass. - uint8_t EffectiveType = PointerEncoding & 0xf; - if (EffectiveType == DW_EH_PE_absptr) - EffectiveType = (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + if (PointerEncoding == DW_EH_PE_omit) + return nullptr; + + // If there's already an edge here then just skip the encoded pointer and + // return the edge's target. + { + auto EdgeI = BlockEdges.find(PointerFieldOffset); + if (EdgeI != BlockEdges.end()) { + LLVM_DEBUG({ + dbgs() << " Existing edge at " + << (BlockToFix.getAddress() + PointerFieldOffset) << " to " + << FieldName << " at " << EdgeI->second.Target->getAddress(); + if (EdgeI->second.Target->hasName()) + dbgs() << " (" << EdgeI->second.Target->getName() << ")"; + dbgs() << "\n"; + }); + if (auto Err = skipEncodedPointer(PointerEncoding, RecordReader)) + return std::move(Err); + return EdgeI->second.Target; + } + } + + // Switch absptr to corresponding udata encoding. + if ((PointerEncoding & 0xf) == DW_EH_PE_absptr) + PointerEncoding |= (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; - orc::ExecutorAddr Addr; - Edge::Kind PointerEdgeKind = Edge::Invalid; - switch (EffectiveType) { + // We need to create an edge. Start by reading the field value. + uint64_t FieldValue; + bool Is64Bit = false; + switch (PointerEncoding & 0xf) { case DW_EH_PE_udata4: { uint32_t Val; if (auto Err = RecordReader.readInteger(Val)) return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta32; - break; - } - case DW_EH_PE_udata8: { - uint64_t Val; - if (auto Err = RecordReader.readInteger(Val)) - return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta64; + FieldValue = Val; break; } case DW_EH_PE_sdata4: { - int32_t Val; + uint32_t Val; if (auto Err = RecordReader.readInteger(Val)) return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta32; + FieldValue = Val; break; } - case DW_EH_PE_sdata8: { - int64_t Val; - if (auto Err = RecordReader.readInteger(Val)) + case DW_EH_PE_udata8: + case DW_EH_PE_sdata8: + Is64Bit = true; + if (auto Err = RecordReader.readInteger(FieldValue)) return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta64; break; - } + default: + llvm_unreachable("Unsupported encoding"); } - if (PointerEdgeKind == Edge::Invalid) - return make_error( - "Unspported edge kind for encoded pointer at " + - formatv("{0:x}", PointerFieldAddress)); + // Find the edge target and edge kind to use. + orc::ExecutorAddr Target; + Edge::Kind PtrEdgeKind = Edge::Invalid; + if ((PointerEncoding & 0x70) == DW_EH_PE_pcrel) { + Target = BlockToFix.getAddress() + PointerFieldOffset; + PtrEdgeKind = Is64Bit ? Delta64 : Delta32; + } else + PtrEdgeKind = Is64Bit ? Pointer64 : Pointer32; + Target += FieldValue; + + // Find or create a symbol to point the edge at. + auto TargetSym = getOrCreateSymbol(PC, Target); + if (!TargetSym) + return TargetSym.takeError(); + BlockToFix.addEdge(PtrEdgeKind, PointerFieldOffset, *TargetSym, 0); - return std::make_pair(Addr, Delta64); + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << (BlockToFix.getAddress() + PointerFieldOffset) << " to " + << FieldName << " at " << TargetSym->getAddress(); + if (TargetSym->hasName()) + dbgs() << " (" << TargetSym->getName() << ")"; + dbgs() << "\n"; + }); + + return &*TargetSym; } Expected EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, orc::ExecutorAddr Addr) { - Symbol *CanonicalSym = nullptr; - - auto UpdateCanonicalSym = [&](Symbol *Sym) { - if (!CanonicalSym || Sym->getLinkage() < CanonicalSym->getLinkage() || - Sym->getScope() < CanonicalSym->getScope() || - (Sym->hasName() && !CanonicalSym->hasName()) || - Sym->getName() < CanonicalSym->getName()) - CanonicalSym = Sym; - }; - - if (auto *SymbolsAtAddr = PC.AddrToSyms.getSymbolsAt(Addr)) - for (auto *Sym : *SymbolsAtAddr) - UpdateCanonicalSym(Sym); - - // If we found an existing symbol at the given address then use it. - if (CanonicalSym) - return *CanonicalSym; + // See whether we have a canonical symbol for the given address already. + auto CanonicalSymI = PC.AddrToSym.find(Addr); + if (CanonicalSymI != PC.AddrToSym.end()) + return *CanonicalSymI->second; // Otherwise search for a block covering the address and create a new symbol. auto *B = PC.AddrToBlock.getBlockCovering(Addr); @@ -730,7 +595,10 @@ Expected EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, return make_error("No symbol or block covering address " + formatv("{0:x16}", Addr)); - return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); + auto &S = + PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); + PC.AddrToSym[S.getAddress()] = &S; + return S; } char EHFrameNullTerminator::NullTerminatorBlockContent[4] = {0, 0, 0, 0}; diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index ef4b47b9aa28ca..55cf7fc63ee795 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,27 +21,16 @@ namespace llvm { namespace jitlink { -/// A LinkGraph pass that splits blocks in an eh-frame section into sub-blocks -/// representing individual eh-frames. -/// EHFrameSplitter should not be run without EHFrameEdgeFixer, which is -/// responsible for adding FDE-to-CIE edges. -class EHFrameSplitter { -public: - EHFrameSplitter(StringRef EHFrameSectionName); - Error operator()(LinkGraph &G); - -private: - Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); - - StringRef EHFrameSectionName; -}; - /// A LinkGraph pass that adds missing FDE-to-CIE, FDE-to-PC and FDE-to-LSDA /// edges. class EHFrameEdgeFixer { public: + /// Create an eh-frame edge fixer. + /// If a given edge-kind is not supported on the target architecture then + /// Edge::Invalid should be used. EHFrameEdgeFixer(StringRef EHFrameSectionName, unsigned PointerSize, - Edge::Kind Delta64, Edge::Kind Delta32, + Edge::Kind Pointer32, Edge::Kind Pointer64, + Edge::Kind Delta32, Edge::Kind Delta64, Edge::Kind NegDelta32); Error operator()(LinkGraph &G); @@ -57,9 +46,10 @@ class EHFrameEdgeFixer { CIEInformation() = default; CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} Symbol *CIESymbol = nullptr; - bool FDEsHaveLSDAField = false; - uint8_t FDEPointerEncoding = 0; - uint8_t LSDAPointerEncoding = 0; + bool AugmentationDataPresent = false; + bool LSDAPresent = false; + uint8_t LSDAEncoding = 0; + uint8_t AddressEncoding = 0; }; struct EdgeTarget { @@ -87,33 +77,38 @@ class EHFrameEdgeFixer { LinkGraph &G; CIEInfosMap CIEInfos; BlockAddressMap AddrToBlock; - SymbolAddressMap AddrToSyms; + DenseMap AddrToSym; }; Error processBlock(ParseContext &PC, Block &B); Error processCIE(ParseContext &PC, Block &B, size_t RecordOffset, - size_t RecordLength, size_t CIEDeltaFieldOffset); + size_t RecordLength, size_t CIEDeltaFieldOffset, + const BlockEdgeMap &BlockEdges); Error processFDE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset, - uint32_t CIEDelta, BlockEdgeMap &BlockEdges); + uint32_t CIEDelta, const BlockEdgeMap &BlockEdges); Expected parseAugmentationString(BinaryStreamReader &RecordReader); - static bool isSupportedPointerEncoding(uint8_t PointerEncoding); - unsigned getPointerEncodingDataSize(uint8_t PointerEncoding); - Expected> - readEncodedPointer(uint8_t PointerEncoding, - orc::ExecutorAddr PointerFieldAddress, - BinaryStreamReader &RecordReader); + Expected readPointerEncoding(BinaryStreamReader &RecordReader, + Block &InBlock, const char *FieldName); + Error skipEncodedPointer(uint8_t PointerEncoding, + BinaryStreamReader &RecordReader); + Expected getOrCreateEncodedPointerEdge( + ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding, + BinaryStreamReader &RecordReader, Block &BlockToFix, + size_t PointerFieldOffset, const char *FieldName); Expected getOrCreateSymbol(ParseContext &PC, orc::ExecutorAddr Addr); StringRef EHFrameSectionName; unsigned PointerSize; - Edge::Kind Delta64; + Edge::Kind Pointer32; + Edge::Kind Pointer64; Edge::Kind Delta32; + Edge::Kind Delta64; Edge::Kind NegDelta32; }; diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index dd3eb97c21a0ea..652ad2a0736e47 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -11,11 +11,14 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" +#include "EHFrameSupportImpl.h" #include "ELFLinkGraphBuilder.h" #include "JITLinkGeneric.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/MathExtras.h" #define DEBUG_TYPE "jitlink" @@ -23,8 +26,7 @@ using namespace llvm; using namespace llvm::jitlink; -namespace llvm { -namespace jitlink { +namespace { class ELFJITLinker_aarch64 : public JITLinker { friend class JITLinker; @@ -37,50 +39,77 @@ class ELFJITLinker_aarch64 : public JITLinker { private: Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { - using namespace aarch64; - using namespace llvm::support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - auto FixupAddress = B.getAddress() + E.getOffset(); - switch (E.getKind()) { - case aarch64::R_AARCH64_CALL26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Call-inst is not 32-bit aligned"); - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Call target is not 32-bit aligned"); - - if (!isInt<28>(Value)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - } - return Error::success(); + return aarch64::applyFixup(G, B, E); } }; template class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { private: - static Expected + enum ELFAArch64RelocationKind : Edge::Kind { + ELFCall26 = Edge::FirstRelocation, + ELFAdrPage21, + ELFAddAbs12, + ELFLdSt8Abs12, + ELFLdSt16Abs12, + ELFLdSt32Abs12, + ELFLdSt64Abs12, + ELFLdSt128Abs12, + ELFMovwAbsG0, + ELFMovwAbsG1, + ELFMovwAbsG2, + ELFMovwAbsG3, + ELFAbs64, + ELFPrel32, + ELFPrel64, + ELFAdrGOTPage21, + ELFLd64GOTLo12, + }; + + static Expected getRelocationKind(const uint32_t Type) { using namespace aarch64; switch (Type) { case ELF::R_AARCH64_CALL26: - return EdgeKind_aarch64::R_AARCH64_CALL26; + case ELF::R_AARCH64_JUMP26: + return ELFCall26; + case ELF::R_AARCH64_ADR_PREL_PG_HI21: + return ELFAdrPage21; + case ELF::R_AARCH64_ADD_ABS_LO12_NC: + return ELFAddAbs12; + case ELF::R_AARCH64_LDST8_ABS_LO12_NC: + return ELFLdSt8Abs12; + case ELF::R_AARCH64_LDST16_ABS_LO12_NC: + return ELFLdSt16Abs12; + case ELF::R_AARCH64_LDST32_ABS_LO12_NC: + return ELFLdSt32Abs12; + case ELF::R_AARCH64_LDST64_ABS_LO12_NC: + return ELFLdSt64Abs12; + case ELF::R_AARCH64_LDST128_ABS_LO12_NC: + return ELFLdSt128Abs12; + case ELF::R_AARCH64_MOVW_UABS_G0_NC: + return ELFMovwAbsG0; + case ELF::R_AARCH64_MOVW_UABS_G1_NC: + return ELFMovwAbsG1; + case ELF::R_AARCH64_MOVW_UABS_G2_NC: + return ELFMovwAbsG2; + case ELF::R_AARCH64_MOVW_UABS_G3: + return ELFMovwAbsG3; + case ELF::R_AARCH64_ABS64: + return ELFAbs64; + case ELF::R_AARCH64_PREL32: + return ELFPrel32; + case ELF::R_AARCH64_PREL64: + return ELFPrel64; + case ELF::R_AARCH64_ADR_GOT_PAGE: + return ELFAdrGOTPage21; + case ELF::R_AARCH64_LD64_GOT_LO12_NC: + return ELFLd64GOTLo12; } - return make_error("Unsupported aarch64 relocation:" + - formatv("{0:d}", Type)); + return make_error( + "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_AARCH64, Type)); } Error addRelocations() override { @@ -99,6 +128,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Error addSingleRelocation(const typename ELFT::Rela &Rel, const typename ELFT::Shdr &FixupSect, Block &BlockToFix) { + using support::ulittle32_t; using Base = ELFLinkGraphBuilder; uint32_t SymbolIndex = Rel.getSymbol(false); @@ -116,18 +146,159 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { inconvertibleErrorCode()); uint32_t Type = Rel.getType(false); - Expected Kind = getRelocationKind(Type); - if (!Kind) - return Kind.takeError(); + Expected RelocKind = getRelocationKind(Type); + if (!RelocKind) + return RelocKind.takeError(); int64_t Addend = Rel.r_addend; orc::ExecutorAddr FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); - Edge GE(*Kind, Offset, *GraphSymbol, Addend); + + // Get a pointer to the fixup content. + const void *FixupContent = BlockToFix.getContent().data() + + (FixupAddress - BlockToFix.getAddress()); + + Edge::Kind Kind = Edge::Invalid; + + switch (*RelocKind) { + case ELFCall26: { + Kind = aarch64::Branch26; + break; + } + case ELFAdrPage21: { + Kind = aarch64::Page21; + break; + } + case ELFAddAbs12: { + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt8Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 0) + return make_error( + "R_AARCH64_LDST8_ABS_LO12_NC target is not a " + "LDRB/STRB (imm12) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt16Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 1) + return make_error( + "R_AARCH64_LDST16_ABS_LO12_NC target is not a " + "LDRH/STRH (imm12) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt32Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 2) + return make_error( + "R_AARCH64_LDST32_ABS_LO12_NC target is not a " + "LDR/STR (imm12, 32 bit) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt64Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 3) + return make_error( + "R_AARCH64_LDST64_ABS_LO12_NC target is not a " + "LDR/STR (imm12, 64 bit) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt128Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 4) + return make_error( + "R_AARCH64_LDST128_ABS_LO12_NC target is not a " + "LDR/STR (imm12, 128 bit) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFMovwAbsG0: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 0) + return make_error( + "R_AARCH64_MOVW_UABS_G0_NC target is not a " + "MOVK/MOVZ (imm16, LSL #0) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFMovwAbsG1: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 16) + return make_error( + "R_AARCH64_MOVW_UABS_G1_NC target is not a " + "MOVK/MOVZ (imm16, LSL #16) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFMovwAbsG2: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 32) + return make_error( + "R_AARCH64_MOVW_UABS_G2_NC target is not a " + "MOVK/MOVZ (imm16, LSL #32) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFMovwAbsG3: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 48) + return make_error( + "R_AARCH64_MOVW_UABS_G3 target is not a " + "MOVK/MOVZ (imm16, LSL #48) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFAbs64: { + Kind = aarch64::Pointer64; + break; + } + case ELFPrel32: { + Kind = aarch64::Delta32; + break; + } + case ELFPrel64: { + Kind = aarch64::Delta64; + break; + } + case ELFAdrGOTPage21: { + Kind = aarch64::GOTPage21; + break; + } + case ELFLd64GOTLo12: { + Kind = aarch64::GOTPageOffset12; + break; + } + }; + + Edge GE(Kind, Offset, *GraphSymbol, Addend); LLVM_DEBUG({ dbgs() << " "; - printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(*Kind)); + printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind)); dbgs() << "\n"; }); @@ -135,6 +306,48 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return Error::success(); } + /// Return the string name of the given ELF aarch64 edge kind. + const char *getELFAArch64RelocationKindName(Edge::Kind R) { + switch (R) { + case ELFCall26: + return "ELFCall26"; + case ELFAdrPage21: + return "ELFAdrPage21"; + case ELFAddAbs12: + return "ELFAddAbs12"; + case ELFLdSt8Abs12: + return "ELFLdSt8Abs12"; + case ELFLdSt16Abs12: + return "ELFLdSt16Abs12"; + case ELFLdSt32Abs12: + return "ELFLdSt32Abs12"; + case ELFLdSt64Abs12: + return "ELFLdSt64Abs12"; + case ELFLdSt128Abs12: + return "ELFLdSt128Abs12"; + case ELFMovwAbsG0: + return "ELFMovwAbsG0"; + case ELFMovwAbsG1: + return "ELFMovwAbsG1"; + case ELFMovwAbsG2: + return "ELFMovwAbsG2"; + case ELFMovwAbsG3: + return "ELFMovwAbsG3"; + case ELFAbs64: + return "ELFAbs64"; + case ELFPrel32: + return "ELFPrel32"; + case ELFPrel64: + return "ELFPrel64"; + case ELFAdrGOTPage21: + return "ELFAdrGOTPage21"; + case ELFLd64GOTLo12: + return "ELFLd64GOTLo12"; + default: + return getGenericEdgeKindName(static_cast(R)); + } + } + public: ELFLinkGraphBuilder_aarch64(StringRef FileName, const object::ELFFile &Obj, const Triple T) @@ -142,6 +355,20 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { aarch64::getEdgeKindName) {} }; +Error buildTables_ELF_aarch64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + aarch64::GOTTableManager GOT; + aarch64::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} + +} // namespace + +namespace llvm { +namespace jitlink { + Expected> createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { LLVM_DEBUG({ @@ -168,11 +395,22 @@ void link_ELF_aarch64(std::unique_ptr G, PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add eh-frame passses. + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, + aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); + + // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/Stubs build pass. + Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); } + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index f83001417e9462..2534498da136db 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -454,8 +454,9 @@ class ELFLinkGraphBuilder_riscv : public ELFLinkGraphBuilder { return EdgeKind_riscv::R_RISCV_32_PCREL; } - return make_error("Unsupported riscv relocation:" + - formatv("{0:d}", Type)); + return make_error( + "Unsupported riscv relocation:" + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_RISCV, Type)); } Error addRelocations() override { diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 79d2cdbb30f18c..8f21274bd1a3c0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/JITLink/TableManager.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" @@ -96,17 +97,6 @@ Error buildTables_ELF_x86_64(LinkGraph &G) { } } // namespace -static const char *getELFX86_64RelocName(uint32_t Type) { - switch (Type) { -#define ELF_RELOC(Name, Number) \ - case Number: \ - return #Name; -#include "llvm/BinaryFormat/ELFRelocs/x86_64.def" -#undef ELF_RELOC - } - return "Unrecognized ELF/x86-64 relocation type"; -} - namespace llvm { namespace jitlink { @@ -145,9 +135,9 @@ class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder { case ELF::R_X86_64_TLSGD: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV; } - return make_error("Unsupported x86-64 relocation type " + - formatv("{0:d}: ", Type) + - getELFX86_64RelocName(Type)); + return make_error( + "Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_X86_64, Type)); } Error addRelocations() override { @@ -379,10 +369,10 @@ void link_ELF_x86_64(std::unique_ptr G, if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { - Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); - Config.PrePrunePasses.push_back( - EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64, - x86_64::Delta32, x86_64::NegDelta32)); + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64, + x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32)); Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); // Construct a JITLinker and run the link function. diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 3ca2e40c7263f8..dd50314d3ed754 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -11,15 +11,15 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" +#include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "MachOLinkGraphBuilder.h" -#include "PerGraphGOTAndPLTStubsBuilder.h" #define DEBUG_TYPE "jitlink" using namespace llvm; using namespace llvm::jitlink; -using namespace llvm::jitlink::MachO_arm64_Edges; namespace { @@ -27,19 +27,39 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin"), - getMachOARM64RelocationKindName), + aarch64::getEdgeKindName), NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: + enum MachOARM64RelocationKind : Edge::Kind { + MachOBranch26 = Edge::FirstRelocation, + MachOPointer32, + MachOPointer64, + MachOPointer64Anon, + MachOPage21, + MachOPageOffset12, + MachOGOTPage21, + MachOGOTPageOffset12, + MachOTLVPage21, + MachOTLVPageOffset12, + MachOPointerToGOT, + MachOPairedAddend, + MachOLDRLiteral19, + MachODelta32, + MachODelta64, + MachONegDelta32, + MachONegDelta64, + }; + static Expected getRelocationKind(const MachO::relocation_info &RI) { switch (RI.r_type) { case MachO::ARM64_RELOC_UNSIGNED: if (!RI.r_pcrel) { if (RI.r_length == 3) - return RI.r_extern ? Pointer64 : Pointer64Anon; + return RI.r_extern ? MachOPointer64 : MachOPointer64Anon; else if (RI.r_length == 2) - return Pointer32; + return MachOPointer32; } break; case MachO::ARM64_RELOC_SUBTRACTOR: @@ -48,46 +68,46 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { // They may be turned into NegDelta by parsePairRelocation. if (!RI.r_pcrel && RI.r_extern) { if (RI.r_length == 2) - return Delta32; + return MachODelta32; else if (RI.r_length == 3) - return Delta64; + return MachODelta64; } break; case MachO::ARM64_RELOC_BRANCH26: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return Branch26; + return MachOBranch26; break; case MachO::ARM64_RELOC_PAGE21: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return Page21; + return MachOPage21; break; case MachO::ARM64_RELOC_PAGEOFF12: if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PageOffset12; + return MachOPageOffset12; break; case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return GOTPage21; + return MachOGOTPage21; break; case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return GOTPageOffset12; + return MachOGOTPageOffset12; break; case MachO::ARM64_RELOC_POINTER_TO_GOT: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PointerToGOT; + return MachOPointerToGOT; break; case MachO::ARM64_RELOC_ADDEND: if (!RI.r_pcrel && !RI.r_extern && RI.r_length == 2) - return PairedAddend; + return MachOPairedAddend; break; case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return TLVPage21; + return MachOTLVPage21; break; case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return TLVPageOffset12; + return MachOTLVPageOffset12; break; } @@ -101,8 +121,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { ", length=" + formatv("{0:d}", RI.r_length)); } - using PairRelocInfo = - std::tuple; + using PairRelocInfo = std::tuple; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. @@ -114,8 +133,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { object::relocation_iterator &RelEnd) { using namespace support; - assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || - (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + assert(((SubtractorKind == MachODelta32 && SubRI.r_length == 2) || + (SubtractorKind == MachODelta64 && SubRI.r_length == 3)) && "Subtractor kind should match length"); assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); @@ -165,17 +184,18 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { FixupValue -= ToSymbol->getAddress().getValue(); } - MachOARM64RelocationKind DeltaKind; + Edge::Kind DeltaKind; Symbol *TargetSymbol; uint64_t Addend; if (&BlockToFix == &FromSymbol->getAddressable()) { TargetSymbol = ToSymbol; - DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + DeltaKind = (SubRI.r_length == 3) ? aarch64::Delta64 : aarch64::Delta32; Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. } else if (&BlockToFix == &ToSymbol->getAddressable()) { TargetSymbol = &*FromSymbol; - DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + DeltaKind = + (SubRI.r_length == 3) ? aarch64::NegDelta64 : aarch64::NegDelta32; Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { // BlockToFix was neither FromSymbol nor ToSymbol. @@ -229,9 +249,9 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { MachO::relocation_info RI = getRelocationInfo(RelItr); // Validate the relocation kind. - auto Kind = getRelocationKind(RI); - if (!Kind) - return Kind.takeError(); + auto MachORelocKind = getRelocationKind(RI); + if (!MachORelocKind) + return MachORelocKind.takeError(); // Find the address of the value to fix up. orc::ExecutorAddr FixupAddress = @@ -255,6 +275,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { return make_error( "Relocation content extends past end of fixup block"); + Edge::Kind Kind = Edge::Invalid; + // Get a pointer to the fixup content. const char *FixupContent = BlockToFix->getContent().data() + (FixupAddress - BlockToFix->getAddress()); @@ -263,7 +285,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; - if (*Kind == PairedAddend) { + if (*MachORelocKind == MachOPairedAddend) { // If this is an Addend relocation then process it and move to the // paired reloc. @@ -275,19 +297,21 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { ++RelItr; RI = getRelocationInfo(RelItr); - Kind = getRelocationKind(RI); - if (!Kind) - return Kind.takeError(); + MachORelocKind = getRelocationKind(RI); + if (!MachORelocKind) + return MachORelocKind.takeError(); - if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12) + if (*MachORelocKind != MachOBranch26 && + *MachORelocKind != MachOPage21 && + *MachORelocKind != MachOPageOffset12) return make_error( "Invalid relocation pair: Addend + " + - StringRef(getMachOARM64RelocationKindName(*Kind))); + StringRef(getMachOARM64RelocationKindName(*MachORelocKind))); LLVM_DEBUG({ dbgs() << " Addend: value = " << formatv("{0:x6}", Addend) - << ", pair is " << getMachOARM64RelocationKindName(*Kind) - << "\n"; + << ", pair is " + << getMachOARM64RelocationKindName(*MachORelocKind) << "\n"; }); // Find the address of the value to fix up. @@ -298,8 +322,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { "different target"); } - switch (*Kind) { - case Branch26: { + switch (*MachORelocKind) { + case MachOBranch26: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -308,23 +332,26 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { if ((Instr & 0x7fffffff) != 0x14000000) return make_error("BRANCH26 target is not a B or BL " "instruction with a zero addend"); + Kind = aarch64::Branch26; break; } - case Pointer32: + case MachOPointer32: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; + Kind = aarch64::Pointer32; break; - case Pointer64: + case MachOPointer64: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; + Kind = aarch64::Pointer64; break; - case Pointer64Anon: { + case MachOPointer64Anon: { orc::ExecutorAddr TargetAddress(*(const ulittle64_t *)FixupContent); auto TargetNSec = findSectionByIndex(RI.r_symbolnum - 1); if (!TargetNSec) @@ -335,11 +362,12 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { else return TargetSymbolOrErr.takeError(); Addend = TargetAddress - TargetSymbol->getAddress(); + Kind = aarch64::Pointer64Anon; break; } - case Page21: - case TLVPage21: - case GOTPage21: { + case MachOPage21: + case MachOTLVPage21: + case MachOGOTPage21: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -349,9 +377,17 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { return make_error("PAGE21/GOTPAGE21 target is not an " "ADRP instruction with a zero " "addend"); + + if (*MachORelocKind == MachOPage21) { + Kind = aarch64::Page21; + } else if (*MachORelocKind == MachOTLVPage21) { + Kind = aarch64::TLVPage21; + } else if (*MachORelocKind == MachOGOTPage21) { + Kind = aarch64::GOTPage21; + } break; } - case PageOffset12: { + case MachOPageOffset12: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -361,10 +397,11 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { if (EncodedAddend != 0) return make_error("GOTPAGEOFF12 target has non-zero " "encoded addend"); + Kind = aarch64::PageOffset12; break; } - case TLVPageOffset12: - case GOTPageOffset12: { + case MachOTLVPageOffset12: + case MachOGOTPageOffset12: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -374,27 +411,35 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { return make_error("GOTPAGEOFF12 target is not an LDR " "immediate instruction with a zero " "addend"); + + if (*MachORelocKind == MachOTLVPageOffset12) { + Kind = aarch64::TLVPageOffset12; + } else if (*MachORelocKind == MachOGOTPageOffset12) { + Kind = aarch64::GOTPageOffset12; + } break; } - case PointerToGOT: + case MachOPointerToGOT: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); + + Kind = aarch64::PointerToGOT; break; - case Delta32: - case Delta64: { + case MachODelta32: + case MachODelta64: { // We use Delta32/Delta64 to represent SUBTRACTOR relocations. // parsePairRelocation handles the paired reloc, and returns the // edge kind to be used (either Delta32/Delta64, or // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, - FixupContent, ++RelItr, RelEnd); + parsePairRelocation(*BlockToFix, *MachORelocKind, RI, + FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + std::tie(Kind, TargetSymbol, Addend) = *PairInfo; assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } @@ -405,108 +450,59 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { LLVM_DEBUG({ dbgs() << " "; - Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *BlockToFix, GE, - getMachOARM64RelocationKindName(*Kind)); + printEdge(dbgs(), *BlockToFix, GE, aarch64::getEdgeKindName(Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); } } return Error::success(); } - unsigned NumSymbols = 0; -}; - -class PerGraphGOTAndPLTStubsBuilder_MachO_arm64 - : public PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_MachO_arm64> { -public: - using PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_MachO_arm64>::PerGraphGOTAndPLTStubsBuilder; - - bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || - E.getKind() == TLVPage21 || E.getKind() == TLVPageOffset12 || - E.getKind() == PointerToGOT; - } - - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); - GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); - } - - void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - if (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || - E.getKind() == TLVPage21 || E.getKind() == TLVPageOffset12) { - // Update the target, but leave the edge addend as-is. - E.setTarget(GOTEntry); - } else if (E.getKind() == PointerToGOT) { - E.setTarget(GOTEntry); - E.setKind(Delta32); - } else - llvm_unreachable("Not a GOT edge?"); - } - - bool isExternalBranchEdge(Edge &E) { - return E.getKind() == Branch26 && !E.getTarget().isDefined(); - } - - Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = G.createContentBlock( - getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); - return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); - } - - void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == Branch26 && "Not a Branch32 edge?"); - assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); - E.setTarget(Stub); - } - -private: - Section &getGOTSection() { - if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); - return *GOTSection; - } - - Section &getStubsSection() { - if (!StubsSection) - StubsSection = - &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); - return *StubsSection; - } - - ArrayRef getGOTEntryBlockContent() { - return {reinterpret_cast(NullGOTEntryContent), - sizeof(NullGOTEntryContent)}; - } - - ArrayRef getStubBlockContent() { - return {reinterpret_cast(StubContent), sizeof(StubContent)}; + /// Return the string name of the given MachO arm64 edge kind. + const char *getMachOARM64RelocationKindName(Edge::Kind R) { + switch (R) { + case MachOBranch26: + return "MachOBranch26"; + case MachOPointer64: + return "MachOPointer64"; + case MachOPointer64Anon: + return "MachOPointer64Anon"; + case MachOPage21: + return "MachOPage21"; + case MachOPageOffset12: + return "MachOPageOffset12"; + case MachOGOTPage21: + return "MachOGOTPage21"; + case MachOGOTPageOffset12: + return "MachOGOTPageOffset12"; + case MachOTLVPage21: + return "MachOTLVPage21"; + case MachOTLVPageOffset12: + return "MachOTLVPageOffset12"; + case MachOPointerToGOT: + return "MachOPointerToGOT"; + case MachOPairedAddend: + return "MachOPairedAddend"; + case MachOLDRLiteral19: + return "MachOLDRLiteral19"; + case MachODelta32: + return "MachODelta32"; + case MachODelta64: + return "MachODelta64"; + case MachONegDelta32: + return "MachONegDelta32"; + case MachONegDelta64: + return "MachONegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); + } } - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[8]; - Section *GOTSection = nullptr; - Section *StubsSection = nullptr; -}; - -const uint8_t - PerGraphGOTAndPLTStubsBuilder_MachO_arm64::NullGOTEntryContent[8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_arm64::StubContent[8] = { - 0x10, 0x00, 0x00, 0x58, // LDR x16, - 0x00, 0x02, 0x1f, 0xd6 // BR x16 + unsigned NumSymbols = 0; }; } // namespace @@ -514,6 +510,15 @@ const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_arm64::StubContent[8] = { namespace llvm { namespace jitlink { +Error buildTables_MachO_arm64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + aarch64::GOTTableManager GOT; + aarch64::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} + class MachOJITLinker_arm64 : public JITLinker { friend class JITLinker; @@ -524,162 +529,8 @@ class MachOJITLinker_arm64 : public JITLinker { : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - - static unsigned getPageOffset12Shift(uint32_t Instr) { - constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; - constexpr uint32_t Vec128Mask = 0x04800000; - - if ((Instr & LoadStoreImm12Mask) == 0x39000000) { - uint32_t ImplicitShift = Instr >> 30; - if (ImplicitShift == 0) - if ((Instr & Vec128Mask) == Vec128Mask) - ImplicitShift = 4; - - return ImplicitShift; - } - - return 0; - } - Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { - using namespace support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Branch-inst is not 32-bit aligned"); - - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Branch26 target is not 32-bit " - "aligned"); - - if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - if (Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case Page21: - case TLVPage21: - case GOTPage21: { - assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && - "GOTPAGE21 with non-zero addend"); - uint64_t TargetPage = - (E.getTarget().getAddress().getValue() + E.getAddend()) & - ~static_cast(4096 - 1); - uint64_t PCPage = - FixupAddress.getValue() & ~static_cast(4096 - 1); - - int64_t PageDelta = TargetPage - PCPage; - if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xffffffe0) == 0x90000000 && - "RawInstr isn't an ADRP instruction"); - uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; - uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; - uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case PageOffset12: { - uint64_t TargetOffset = - (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - unsigned ImmShift = getPageOffset12Shift(RawInstr); - - if (TargetOffset & ((1 << ImmShift) - 1)) - return make_error("PAGEOFF12 target is not aligned"); - - uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case TLVPageOffset12: - case GOTPageOffset12: { - assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xfffffc00) == 0xf9400000 && - "RawInstr isn't a 64-bit LDR immediate"); - - uint32_t TargetOffset = E.getTarget().getAddress().getValue() & 0xfff; - assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); - uint32_t EncodedImm = (TargetOffset >> 3) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case LDRLiteral19: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "LDR is not 32-bit aligned"); - assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); - int64_t Delta = E.getTarget().getAddress() - FixupAddress; - if (Delta & 0x3) - return make_error("LDR literal target is not 32-bit " - "aligned"); - if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t EncodedImm = - ((static_cast(Delta) >> 2) & 0x7ffff) << 5; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - default: - llvm_unreachable("Unrecognized edge kind"); - } - - return Error::success(); + return aarch64::applyFixup(G, B, E); } uint64_t NullValue = 0; @@ -712,13 +563,14 @@ void link_MachO_arm64(std::unique_ptr G, // Add eh-frame passses. // FIXME: Prune eh-frames for which compact-unwind is available once // we support compact-unwind registration with libunwind. - Config.PrePrunePasses.push_back(EHFrameSplitter("__TEXT,__eh_frame")); Config.PrePrunePasses.push_back( - EHFrameEdgeFixer("__TEXT,__eh_frame", 8, Delta64, Delta32, NegDelta32)); + DWARFRecordSectionSplitter("__TEXT,__eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + "__TEXT,__eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, + aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back( - PerGraphGOTAndPLTStubsBuilder_MachO_arm64::asPass); + Config.PostPrunePasses.push_back(buildTables_MachO_arm64); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) @@ -728,44 +580,5 @@ void link_MachO_arm64(std::unique_ptr G, MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); } -const char *getMachOARM64RelocationKindName(Edge::Kind R) { - switch (R) { - case Branch26: - return "Branch26"; - case Pointer64: - return "Pointer64"; - case Pointer64Anon: - return "Pointer64Anon"; - case Page21: - return "Page21"; - case PageOffset12: - return "PageOffset12"; - case GOTPage21: - return "GOTPage21"; - case GOTPageOffset12: - return "GOTPageOffset12"; - case TLVPage21: - return "TLVPage21"; - case TLVPageOffset12: - return "TLVPageOffset12"; - case PointerToGOT: - return "PointerToGOT"; - case PairedAddend: - return "PairedAddend"; - case LDRLiteral19: - return "LDRLiteral19"; - case Delta32: - return "Delta32"; - case Delta64: - return "Delta64"; - case NegDelta32: - return "NegDelta32"; - case NegDelta64: - return "NegDelta64"; - default: - return getGenericEdgeKindName(static_cast(R)); - } -} - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 82afaa3aa3c554..2f61b31ae38328 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "MachOLinkGraphBuilder.h" @@ -504,12 +505,13 @@ void link_MachO_x86_64(std::unique_ptr G, } LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64() { - return EHFrameSplitter("__TEXT,__eh_frame"); + return DWARFRecordSectionSplitter("__TEXT,__eh_frame"); } LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64() { return EHFrameEdgeFixer("__TEXT,__eh_frame", x86_64::PointerSize, - x86_64::Delta64, x86_64::Delta32, x86_64::NegDelta32); + x86_64::Pointer32, x86_64::Pointer64, x86_64::Delta32, + x86_64::Delta64, x86_64::NegDelta32); } } // end namespace jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 6dccc481188513..28a6f9ce90d926 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,13 +18,55 @@ namespace llvm { namespace jitlink { namespace aarch64 { -const char *getEdgeKindName(Edge::Kind K) { - switch (K) { - case R_AARCH64_CALL26: - return "R_AARCH64_CALL26"; +const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; + +const uint8_t StubContent[8] = { + 0x10, 0x00, 0x00, 0x58, // LDR x16, + 0x00, 0x02, 0x1f, 0xd6 // BR x16 +}; + +const char *getEdgeKindName(Edge::Kind R) { + switch (R) { + case Branch26: + return "Branch26"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case Page21: + return "Page21"; + case PageOffset12: + return "PageOffset12"; + case MoveWide16: + return "MoveWide16"; + case GOTPage21: + return "GOTPage21"; + case GOTPageOffset12: + return "GOTPageOffset12"; + case TLVPage21: + return "TLVPage21"; + case TLVPageOffset12: + return "TLVPageOffset12"; + case PointerToGOT: + return "PointerToGOT"; + case PairedAddend: + return "PairedAddend"; + case LDRLiteral19: + return "LDRLiteral19"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); } - return getGenericEdgeKindName(K); } + } // namespace aarch64 } // namespace jitlink } // namespace llvm diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp index aba2ad31553560..35f1ba3364594e 100644 --- a/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -686,14 +686,12 @@ void WinCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm, bool WinCOFFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl( const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB, bool InSet, bool IsPCRel) const { - // Don't drop relocations between functions, even if they are in the same text - // section. Multiple Visual C++ linker features depend on having the - // relocations present. The /INCREMENTAL flag will cause these relocations to - // point to thunks, and the /GUARD:CF flag assumes that it can use relocations - // to approximate the set of all address taken functions. LLD's implementation - // of /GUARD:CF also relies on the existance of these relocations. + // MS LINK expects to be able to replace all references to a function with a + // thunk to implement their /INCREMENTAL feature. Make sure we don't optimize + // away any relocations to functions. uint16_t Type = cast(SymA).getType(); - if ((Type >> COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION) + if (Asm.isIncrementalLinkerCompatible() && + (Type >> COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION) return false; return MCObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(Asm, SymA, FB, InSet, IsPCRel); diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index dc5e6807945d72..a80c08b94948ec 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -4504,23 +4504,6 @@ bool AArch64FastISel::selectIntExt(const Instruction *I) { // Try to optimize already sign-/zero-extended values from function arguments. bool IsZExt = isa(I); - if (const auto *Arg = dyn_cast(I->getOperand(0))) { - if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) { - if (RetVT == MVT::i64 && SrcVT != MVT::i64) { - Register ResultReg = createResultReg(&AArch64::GPR64RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, - TII.get(AArch64::SUBREG_TO_REG), ResultReg) - .addImm(0) - .addReg(SrcReg) - .addImm(AArch64::sub_32); - SrcReg = ResultReg; - } - - updateValueMap(I, SrcReg); - return true; - } - } - unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt); if (!ResultReg) return false; diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp index 180012198c42cf..54eac4d857f15f 100644 --- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp @@ -430,7 +430,9 @@ static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { // with them. if (Addr) { Type *PtrTy = cast(Addr->getType()->getScalarType()); - if (PtrTy->getPointerAddressSpace() != 0) + auto AS = PtrTy->getPointerAddressSpace(); + // Allow for custom addresspaces + if (AS != 0 && AS < 10) return false; } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 46ff0994e04e7f..e629598ad36b7d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4892,6 +4892,27 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened( if (hasIrregularType(ScalarTy, DL)) return false; + // If the group involves a non-integral pointer, we may not be able to + // losslessly cast all values to a common type. + unsigned InterleaveFactor = Group->getFactor(); + bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy); + for (unsigned i = 0; i < InterleaveFactor; i++) { + Instruction *Member = Group->getMember(i); + if (!Member) + continue; + auto *MemberTy = getLoadStoreType(Member); + bool MemberNI = DL.isNonIntegralPointerType(MemberTy); + // Don't coerce non-integral pointers to integers or vice versa. + if (MemberNI != ScalarNI) { + // TODO: Consider adding special nullptr value case here + return false; + } else if (MemberNI && ScalarNI && + ScalarTy->getPointerAddressSpace() != + MemberTy->getPointerAddressSpace()) { + return false; + } + } + // Check if masking is required. // A Group may need masking for one of two reasons: it resides in a block that // needs predication, or it was decided to use masking to deal with gaps diff --git a/llvm/test/CodeGen/PowerPC/issue55983.ll b/llvm/test/CodeGen/PowerPC/issue55983.ll new file mode 100644 index 00000000000000..7ebde001ada7f4 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/issue55983.ll @@ -0,0 +1,66 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -opaque-pointers -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s +; RUN: llc -opaque-pointers -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s + +define void @foo() #0 { +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %bb0 +; CHECK-NEXT: bc 12, 20, .LBB0_2 +; CHECK-NEXT: # %bb.1: # %bb1 +; CHECK-NEXT: ld 3, 0(3) +; CHECK-NEXT: cmpd 7, 3, 3 +; CHECK-NEXT: bne- 7, .+4 +; CHECK-NEXT: isync +; CHECK-NEXT: .LBB0_2: # %bb2 +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic {}*, {}** undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @bar() #0 { +; CHECK-LABEL: bar: +; CHECK: # %bb.0: # %bb0 +; CHECK-NEXT: bc 12, 20, .LBB1_2 +; CHECK-NEXT: # %bb.1: # %bb1 +; CHECK-NEXT: ld 3, 0(3) +; CHECK-NEXT: cmpd 7, 3, 3 +; CHECK-NEXT: bne- 7, .+4 +; CHECK-NEXT: isync +; CHECK-NEXT: .LBB1_2: # %bb2 +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic ptr, ptr undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @foobar() { +; CHECK-LABEL: foobar: +; CHECK: # %bb.0: # %top +; CHECK-NEXT: bc 4, 20, .LBB2_2 +; CHECK-NEXT: # %bb.1: # %err86 +; CHECK-NEXT: .LBB2_2: # %pass9 +; CHECK-NEXT: ld 3, 0(3) +; CHECK-NEXT: cmpd 7, 3, 3 +; CHECK-NEXT: bne- 7, .+4 +; CHECK-NEXT: isync +top: + br i1 undef, label %err86, label %pass9 + +pass9: ; preds = %top + %0 = load atomic {} addrspace(10)*, {} addrspace(10)* addrspace(11)* undef acquire, align 8 + unreachable + +err86: ; preds = %top + unreachable +} diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s new file mode 100644 index 00000000000000..ff7e5a6546f6f4 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s @@ -0,0 +1,80 @@ +# REQUIRES: asserts +# RUN: llvm-mc -triple=aarch64-linux-gnu -filetype=obj -o %t %s +# RUN: llvm-jitlink -noexec -phony-externals -debug-only=jitlink %t 2>&1 | \ +# RUN: FileCheck %s +# +# Check that splitting of eh-frame sections works. +# +# CHECK: DWARFRecordSectionSplitter: Processing .eh_frame... +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Extracted {{.*}} section = .eh_frame +# CHECK: Processing CFI record at +# CHECK: Extracted {{.*}} section = .eh_frame +# CHECK: EHFrameEdgeFixer: Processing .eh_frame... +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Record is CIE +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Record is FDE +# CHECK: Adding edge at {{.*}} to CIE at: {{.*}} +# CHECK: Existing edge at {{.*}} to PC begin at {{.*}} +# CHECK: Adding keep-alive edge from target at {{.*}} to FDE at {{.*}} +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Record is FDE +# CHECK: Adding edge at {{.*}} to CIE at: {{.*}} +# CHECK: Existing edge at {{.*}} to PC begin at {{.*}} +# CHECK: Adding keep-alive edge from target at {{.*}} to FDE at {{.*}} + + .text + .globl main + .p2align 2 + .type main,@function +main: + .cfi_startproc + sub sp, sp, #32 + .cfi_def_cfa_offset 32 + stp x29, x30, [sp, #16] + add x29, sp, #16 + .cfi_def_cfa w29, 16 + .cfi_offset w30, -8 + .cfi_offset w29, -16 + stur wzr, [x29, #-4] + mov x0, #4 + bl __cxa_allocate_exception + mov w8, #1 + str w8, [x0] + adrp x1, :got:_ZTIi + ldr x1, [x1, :got_lo12:_ZTIi] + mov x2, xzr + bl __cxa_throw +.main_end: + .size main, .main_end-main + .cfi_endproc + + .globl dup + .p2align 2 + .type dup,@function +dup: + .cfi_startproc + sub sp, sp, #32 + .cfi_def_cfa_offset 32 + stp x29, x30, [sp, #16] + add x29, sp, #16 + .cfi_def_cfa w29, 16 + .cfi_offset w30, -8 + .cfi_offset w29, -16 + stur wzr, [x29, #-4] + mov x0, #4 + bl __cxa_allocate_exception + mov w8, #1 + str w8, [x0] + adrp x1, :got:_ZTIi + ldr x1, [x1, :got_lo12:_ZTIi] + mov x2, xzr + bl __cxa_throw +.dup_end: + .size dup, .dup_end-dup + .cfi_endproc \ No newline at end of file diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test new file mode 100644 index 00000000000000..fa224d856c4efa --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test @@ -0,0 +1,60 @@ +# RUN: yaml2obj -o %t.o %s +# RUN: llvm-jitlink -noexec -check %s %t.o + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_AARCH64 + SectionHeaderStringTable: .strtab +Sections: + - Name: .text + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + AddressAlign: 0x4 + Content: E0031F2AC0035FD6 + - Name: .eh_frame + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + AddressAlign: 0x8 + Content: 1000000000000000017A5200017C1E011B0C1F001000000018000000000000000800000000000000 + - Name: .rela.eh_frame + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .eh_frame + Relocations: + - Offset: 0x1C + Symbol: text + Type: R_AARCH64_PREL32 + - Offset: 0x20 + Symbol: text + Type: R_AARCH64_PREL64 + - Type: SectionHeaderTable + Sections: + - Name: .strtab + - Name: .text + - Name: .eh_frame + - Name: .rela.eh_frame + - Name: .symtab +Symbols: + - Name: text + Type: STT_SECTION + Binding: STB_GLOBAL + Section: .text + Size: 0x8 + - Name: eh_frame + Type: STT_SECTION + Binding: STB_GLOBAL + Section: .eh_frame + Size: 0x28 + - Name: main + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Size: 0x8 + +# jitlink-check: *{4}(eh_frame + 28) = (text + 0) - (eh_frame + 28) +# jitlink-check: *{8}(eh_frame + 32) = (text + 0) - (eh_frame + 32) \ No newline at end of file diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s new file mode 100644 index 00000000000000..3eb72e5b2456aa --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -0,0 +1,257 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false \ +# RUN: -position-independent -filetype=obj -o %t/elf_reloc.o %s +# RUN: llvm-jitlink -noexec \ +# RUN: -abs external_data=0xdeadbeef \ +# RUN: -abs external_func=0xcafef00d \ +# RUN: -check %s %t/elf_reloc.o + + .text + + .globl main + .p2align 2 + .type main,@function +main: + ret + + .size main, .-main + +# Check R_AARCH64_CALL26 / R_AARCH64_JUMP26 relocation of a local function call +# +# jitlink-check: decode_operand(local_func_call26, 0)[25:0] = (local_func - local_func_call26)[27:2] +# jitlink-check: decode_operand(local_func_jump26, 0)[25:0] = (local_func - local_func_jump26)[27:2] + .globl local_func + .p2align 2 + .type local_func,@function +local_func: + ret + .size local_func, .-local_func + + .globl local_func_call26 + .p2align 2 +local_func_call26: + bl local_func + .size local_func_call26, .-local_func_call26 + + .globl local_func_jump26 + .p2align 2 +local_func_jump26: + b local_func + .size local_func_jump26, .-local_func_jump26 + +# Check R_AARCH64_ADR_PREL_PG_HI21 / R_AARCH64_ADD_ABS_LO12_NC relocation of a local symbol +# +# For the ADR_PREL_PG_HI21/ADRP instruction we have the 21-bit delta to the 4k page +# containing the global. +# +# jitlink-check: decode_operand(test_adr_prel, 1) = (named_data - test_adr_prel)[32:12] +# jitlink-check: decode_operand(test_add_abs_lo12, 2) = (named_data + 0)[11:0] + .globl test_adr_prel + .p2align 2 +test_adr_prel: + adrp x0, named_data + .size test_adr_prel, .-test_adr_prel + + .globl test_add_abs_lo12 + .p2align 2 +test_add_abs_lo12: + add x0, x0, :lo12:named_data + .size test_add_abs_lo12, .-test_add_abs_lo12 + +# Check that calls/jumps to external functions trigger the generation of stubs and GOT +# entries. +# +# jitlink-check: decode_operand(test_external_call, 0) = (stub_addr(elf_reloc.o, external_func) - test_external_call)[27:2] +# jitlink-check: decode_operand(test_external_jump, 0) = (stub_addr(elf_reloc.o, external_func) - test_external_jump)[27:2] +# jitlink-check: *{8}(got_addr(elf_reloc.o, external_func)) = external_func + .globl test_external_call + .p2align 2 +test_external_call: + bl external_func + .size test_external_call, .-test_external_call + + .globl test_external_jump + .p2align 2 +test_external_jump: + b external_func + .size test_external_jump, .-test_external_jump + +# Check R_AARCH64_LDST*_ABS_LO12_NC relocation of a local symbol +# +# The immediate value should be the symbol address right shifted according to its instruction bitwidth. +# +# jitlink-check: decode_operand(test_ldrb, 2) = named_data[11:0] +# jitlink-check: decode_operand(test_ldrsb, 2) = (named_data + 0)[11:0] +# jitlink-check: decode_operand(test_ldrh, 2) = (named_data + 0)[11:1] +# jitlink-check: decode_operand(test_ldrsh, 2) = (named_data + 0)[11:1] +# jitlink-check: decode_operand(test_ldr_32bit, 2) = (named_data + 0)[11:2] +# jitlink-check: decode_operand(test_ldr_64bit, 2) = (named_data + 0)[11:3] +# jitlink-check: decode_operand(test_strb, 2) = named_data[11:0] +# jitlink-check: decode_operand(test_strh, 2) = (named_data + 0)[11:1] +# jitlink-check: decode_operand(test_str_32bit, 2) = (named_data + 0)[11:2] +# jitlink-check: decode_operand(test_str_64bit, 2) = (named_data + 0)[11:3] + + .globl test_ldrb +test_ldrb: + ldrb w0, [x1, :lo12:named_data] + .size test_ldrb, .-test_ldrb + + .globl test_ldrsb +test_ldrsb: + ldrsb w0, [x1, :lo12:named_data] + .size test_ldrsb, .-test_ldrsb + + .globl test_ldrh +test_ldrh: + ldrh w0, [x1, :lo12:named_data] + .size test_ldrh, .-test_ldrh + + .globl test_ldrsh +test_ldrsh: + ldrsh w0, [x1, :lo12:named_data] + .size test_ldrsh, .-test_ldrsh + + .globl test_ldr_32bit +test_ldr_32bit: + ldr w0, [x1, :lo12:named_data] + .size test_ldr_32bit, .-test_ldr_32bit + + .globl test_ldr_64bit +test_ldr_64bit: + ldr x0, [x1, :lo12:named_data] + .size test_ldr_64bit, .-test_ldr_64bit + + .globl test_strb +test_strb: + strb w0, [x1, :lo12:named_data] + .size test_strb, .-test_strb + + .globl test_strh +test_strh: + strh w0, [x1, :lo12:named_data] + .size test_strh, .-test_strh + + .globl test_str_32bit +test_str_32bit: + str w0, [x1, :lo12:named_data] + .size test_str_32bit, .-test_str_32bit + + .globl test_str_64bit +test_str_64bit: + str x0, [x1, :lo12:named_data] + .size test_str_64bit, .-test_str_64bit + + +# Check R_AARCH64_MOVW_UABS_G*_NC relocation of a local symbol +# +# The immediate value should be the symbol address right shifted according to LSL value +# +# jitlink-check: decode_operand(test_movz_g0_nc, 1) = named_data[15:0] +# jitlink-check: decode_operand(test_movk_g0_nc, 2) = named_data[15:0] +# jitlink-check: decode_operand(test_movz_g1_nc, 1) = named_data[31:16] +# jitlink-check: decode_operand(test_movk_g1_nc, 2) = named_data[31:16] +# jitlink-check: decode_operand(test_movz_g2_nc, 1) = named_data[47:32] +# jitlink-check: decode_operand(test_movk_g2_nc, 2) = named_data[47:32] +# jitlink-check: decode_operand(test_movz_g3, 1) = named_data[63:48] +# jitlink-check: decode_operand(test_movk_g3, 2) = named_data[63:48] + + .globl test_movz_g0_nc +test_movz_g0_nc: + movz x0, #:abs_g0_nc:named_data + .size test_movz_g0_nc, .-test_movz_g0_nc + + .globl test_movk_g0_nc +test_movk_g0_nc: + movk x0, #:abs_g0_nc:named_data + .size test_movk_g0_nc, .-test_movk_g0_nc + + .globl test_movz_g1_nc +test_movz_g1_nc: + movz x0, #:abs_g1_nc:named_data + .size test_movz_g1_nc, .-test_movz_g1_nc + + .globl test_movk_g1_nc +test_movk_g1_nc: + movk x0, #:abs_g1_nc:named_data + .size test_movk_g1_nc, .-test_movk_g1_nc + + .globl test_movz_g2_nc +test_movz_g2_nc: + movz x0, #:abs_g2_nc:named_data + .size test_movz_g2_nc, .-test_movz_g2_nc + + .globl test_movk_g2_nc +test_movk_g2_nc: + movk x0, #:abs_g2_nc:named_data + .size test_movk_g2_nc, .-test_movk_g2_nc + + .globl test_movk_g3 +test_movk_g3: + movk x0, #:abs_g3:named_data + .size test_movk_g3, .-test_movk_g3 + + .globl test_movz_g3 +test_movz_g3: + movz x0, #:abs_g3:named_data + .size test_movz_g3, .-test_movz_g3 + +# Check R_AARCH64_ABS64 relocation of a function pointer to local symbol +# +# jitlink-check: *{8}local_func_addr_quad = named_func + .globl local_func_addr_quad + .p2align 3 +local_func_addr_quad: + .xword named_func + .size local_func_addr_quad, 8 + +# Check R_AARCH64_ABS64 relocation of a function pointer to external symbol +# +# jitlink-check: *{8}external_func_addr_quad = external_func + .globl external_func_addr_quad + .p2align 3 +external_func_addr_quad: + .xword external_func + .size external_func_addr_quad, 8 + +# Check R_AARCH64_ADR_GOT_PAGE / R_AARCH64_LD64_GOT_LO12_NC handling with a +# reference to an external symbol. Validate both the reference to the GOT entry, +# and also the content of the GOT entry. +# +# For the ADRP :got: instruction we have the 21-bit delta to the 4k page +# containing the GOT entry for external_data. +# +# For the LDR :got_lo12: instruction we have the 12-bit offset of the entry +# within the page. +# +# jitlink-check: *{8}(got_addr(elf_reloc.o, external_data)) = external_data +# jitlink-check: decode_operand(test_adr_gotpage_external, 1) = \ +# jitlink-check: (got_addr(elf_reloc.o, external_data)[32:12] - \ +# jitlink-check: test_adr_gotpage_external[32:12]) +# jitlink-check: decode_operand(test_ld64_gotlo12_external, 2) = \ +# jitlink-check: got_addr(elf_reloc.o, external_data)[11:3] + .globl test_adr_gotpage_external + .p2align 2 +test_adr_gotpage_external: + adrp x0, :got:external_data + .size test_adr_gotpage_external, .-test_adr_gotpage_external + + .globl test_ld64_gotlo12_external + .p2align 2 +test_ld64_gotlo12_external: + ldr x0, [x0, :got_lo12:external_data] + .size test_ld64_gotlo12_external, .-test_ld64_gotlo12_external + + .globl named_data + .p2align 4 + .type named_data,@object +named_data: + .quad 0x2222222222222222 + .quad 0x3333333333333333 + .size named_data, .-named_data + + .globl named_func + .p2align 2 + .type named_func,@function +named_func: + ret + .size named_func, .-named_func diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s index 34b4830d029ae3..6ae5973fb82307 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s @@ -5,7 +5,7 @@ # # Check that splitting of eh-frame sections works. # -# CHECK: EHFrameSplitter: Processing __TEXT,__eh_frame... +# CHECK: DWARFRecordSectionSplitter: Processing __TEXT,__eh_frame... # CHECK: Processing block at # CHECK: Processing CFI record at # CHECK: Extracted {{.*}} section = __TEXT,__eh_frame @@ -19,9 +19,9 @@ # CHECK: Processing CFI record at # CHECK: Record is FDE # CHECK: Adding edge at {{.*}} to CIE at: {{.*}} -# CHECK: Already has edge at {{.*}} to PC at {{.*}} +# CHECK: Existing edge at {{.*}} to PC begin at {{.*}} # CHECK: Adding keep-alive edge from target at {{.*}} to FDE at {{.*}} -# CHECK: Already has edge at {{.*}} to LSDA at {{.*}} +# CHECK: Existing edge at {{.*}} to LSDA at {{.*}} .section __TEXT,__text,regular,pure_instructions .globl _main diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s new file mode 100644 index 00000000000000..83e46f8a12d427 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s @@ -0,0 +1,203 @@ +# REQUIRES: asserts +# UNSUPPORTED: system-windows +# RUN: llvm-mc -triple=x86_64-pc-linux-gnu -large-code-model \ +# RUN: -filetype=obj -o %t %s +# RUN: llvm-jitlink -debug-only=jitlink -noexec %t 2>&1 | FileCheck %s +# +# Check handling of pointer encodings for personality functions when compiling +# with `-mcmodel=large -static`. +# + +# CHECK: Record is CIE +# CHECK-NEXT: edge at {{.*}} to personality at {{.*}} (DW.ref.__gxx_personality_v0) +# CHECK: Record is CIE +# CHECK-NEXT: edge at {{.*}} to personality at {{.*}} (__gxx_personality_v0) + + .text + .file "eh.cpp" + + .globl main + .p2align 4, 0x90 + .type main,@function +main: + xorl %eax, %eax + retq +.Lfunc_end_main: + .size main, .Lfunc_end_main-main + +# pe_absptr uses absptr encoding for __gxx_personality_v0 + .text + .globl pe_absptr + .p2align 4, 0x90 + .type pe_absptr,@function +pe_absptr: +.Lfunc_begin0: + .cfi_startproc + .cfi_personality 0, __gxx_personality_v0 + .cfi_lsda 0, .Lexception0 + pushq %rax + .cfi_def_cfa_offset 16 + movabsq $__cxa_allocate_exception, %rax + movl $4, %edi + callq *%rax + movl $42, (%rax) +.Ltmp0: + movabsq $_ZTIi, %rsi + movabsq $__cxa_throw, %rcx + movq %rax, %rdi + xorl %edx, %edx + callq *%rcx +.Ltmp1: +.LBB0_2: +.Ltmp2: + movabsq $__cxa_begin_catch, %rcx + movq %rax, %rdi + callq *%rcx + movabsq $__cxa_end_catch, %rax + popq %rcx + .cfi_def_cfa_offset 8 + jmpq *%rax +.Lfunc_end0: + .size pe_absptr, .Lfunc_end0-pe_absptr + .cfi_endproc + .section .gcc_except_table,"a",@progbits + .p2align 2 +GCC_except_table0: +.Lexception0: + .byte 255 # @LPStart Encoding = omit + .byte 0 # @TType Encoding = absptr + .uleb128 .Lttbase0-.Lttbaseref0 +.Lttbaseref0: + .byte 1 # Call site Encoding = uleb128 + .uleb128 .Lcst_end0-.Lcst_begin0 +.Lcst_begin0: + .uleb128 .Lfunc_begin0-.Lfunc_begin0 # >> Call Site 1 << + .uleb128 .Ltmp0-.Lfunc_begin0 # Call between .Lfunc_begin0 and .Ltmp0 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup + .uleb128 .Ltmp0-.Lfunc_begin0 # >> Call Site 2 << + .uleb128 .Ltmp1-.Ltmp0 # Call between .Ltmp0 and .Ltmp1 + .uleb128 .Ltmp2-.Lfunc_begin0 # jumps to .Ltmp2 + .byte 1 # On action: 1 + .uleb128 .Ltmp1-.Lfunc_begin0 # >> Call Site 3 << + .uleb128 .Lfunc_end0-.Ltmp1 # Call between .Ltmp1 and .Lfunc_end0 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup +.Lcst_end0: + .byte 1 # >> Action Record 1 << + # Catch TypeInfo 1 + .byte 0 # No further actions + .p2align 2 + # >> Catch TypeInfos << + .quad _ZTIi # TypeInfo 1 +.Lttbase0: + .p2align 2 + # -- End function + +# pe_indir_pcrel_sdata8 uses 0x9C -- Indirect, pc-rel, sdata8 encoding to +# DW.ref.__gxx_personality_v0 + .text + .globl pe_indir_pcrel_sdata8 + .p2align 4, 0x90 + .type pe_indir_pcrel_sdata8,@function +pe_indir_pcrel_sdata8: +.Lfunc_begin1: + .cfi_startproc + .cfi_personality 156, DW.ref.__gxx_personality_v0 + .cfi_lsda 28, .Lexception1 + pushq %r14 + .cfi_def_cfa_offset 16 + pushq %rbx + .cfi_def_cfa_offset 24 + pushq %rax + .cfi_def_cfa_offset 32 + .cfi_offset %rbx, -24 + .cfi_offset %r14, -16 +.L1$pb: + leaq .L1$pb(%rip), %rax + movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rbx + addq %rax, %rbx + movabsq $__cxa_allocate_exception@GOT, %rax + movl $4, %edi + callq *(%rbx,%rax) + movl $42, (%rax) +.Ltmp4: + movabsq $_ZTIi@GOT, %rcx + movq (%rbx,%rcx), %rsi + movabsq $__cxa_throw@GOT, %rcx + movq %rax, %rdi + xorl %edx, %edx + movq %rbx, %r14 + callq *(%rbx,%rcx) +.Ltmp5: +.LBB1_2: +.Ltmp6: + movabsq $__cxa_begin_catch@GOT, %rcx + movq %rax, %rdi + callq *(%r14,%rcx) + movabsq $__cxa_end_catch@GOT, %rax + movq %r14, %rcx + addq $8, %rsp + .cfi_def_cfa_offset 24 + popq %rbx + .cfi_def_cfa_offset 16 + popq %r14 + .cfi_def_cfa_offset 8 + jmpq *(%rcx,%rax) +.Lfunc_end1: + .size pe_indir_pcrel_sdata8, .Lfunc_end1-pe_indir_pcrel_sdata8 + .cfi_endproc + .section .gcc_except_table,"a",@progbits + .p2align 2 +GCC_except_table1: +.Lexception1: + .byte 255 # @LPStart Encoding = omit + .byte 156 # @TType Encoding = indirect pcrel sdata8 + .uleb128 .Lttbase1-.Lttbaseref1 +.Lttbaseref1: + .byte 1 # Call site Encoding = uleb128 + .uleb128 .Lcst_end1-.Lcst_begin1 +.Lcst_begin1: + .uleb128 .Lfunc_begin1-.Lfunc_begin1 # >> Call Site 1 << + .uleb128 .Ltmp4-.Lfunc_begin1 # Call between .Lfunc_begin1 and .Ltmp4 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup + .uleb128 .Ltmp4-.Lfunc_begin1 # >> Call Site 2 << + .uleb128 .Ltmp5-.Ltmp4 # Call between .Ltmp4 and .Ltmp5 + .uleb128 .Ltmp6-.Lfunc_begin1 # jumps to .Ltmp6 + .byte 1 # On action: 1 + .uleb128 .Ltmp5-.Lfunc_begin1 # >> Call Site 3 << + .uleb128 .Lfunc_end1-.Ltmp5 # Call between .Ltmp5 and .Lfunc_end1 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup +.Lcst_end1: + .byte 1 # >> Action Record 1 << + # Catch TypeInfo 1 + .byte 0 # No further actions + .p2align 2 + # >> Catch TypeInfos << +.Ltmp7: # TypeInfo 1 + .quad .L_ZTIi.DW.stub-.Ltmp7 +.Lttbase1: + .p2align 2 + # -- End function + + .data + .p2align 3 +.L_ZTIi.DW.stub: + .quad _ZTIi + .hidden DW.ref.__gxx_personality_v0 + .weak DW.ref.__gxx_personality_v0 + + .section .data.DW.ref.__gxx_personality_v0,"aGw",@progbits,DW.ref.__gxx_personality_v0,comdat + .p2align 3 + .type DW.ref.__gxx_personality_v0,@object + .size DW.ref.__gxx_personality_v0, 8 +DW.ref.__gxx_personality_v0: + .quad __gxx_personality_v0 + + .ident "clang version 13.0.1" + .section ".note.GNU-stack","",@progbits + .addrsig + .addrsig_sym __gxx_personality_v0 + .addrsig_sym _ZTIi diff --git a/llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s b/llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s new file mode 100644 index 00000000000000..da6cb77f970426 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s @@ -0,0 +1,28 @@ +# REQUIRES: asserts +# RUN: llvm-mc -triple=x86_64-apple-macos10.9 -filetype=obj -o %t %s +# RUN: llvm-jitlink -noexec %t +# +# Verify that PC-begin candidate symbols have been sorted correctly when adding +# PC-begin edges for FDEs. In this test both _main and _X are at address zero, +# however we expect to select _main over _X as _X is common. If the sorting +# fails we'll trigger an assert in EHFrameEdgeFixer, otherwise this test will +# succeed. + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 12, 0 sdk_version 13, 0 + .globl _main + .p2align 4, 0x90 +_main: + .cfi_startproc + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + xorl %eax, %eax + popq %rbp + retq + .cfi_endproc + + .comm _X,4,2 +.subsections_via_symbols diff --git a/llvm/test/MC/COFF/diff.s b/llvm/test/MC/COFF/diff.s index 90466b59d02522..640bf8189e0395 100644 --- a/llvm/test/MC/COFF/diff.s +++ b/llvm/test/MC/COFF/diff.s @@ -1,14 +1,19 @@ // RUN: llvm-mc -filetype=obj -triple i686-pc-mingw32 %s | llvm-readobj -S --sr --sd - | FileCheck %s -// COFF resolves differences between labels in the same section, unless that -// label is declared with function type. - .section baz, "xr" + .def X + .scl 2; + .type 32; + .endef .globl X X: mov Y-X+42, %eax retl + .def Y + .scl 2; + .type 32; + .endef .globl Y Y: retl @@ -25,11 +30,6 @@ _foobar: # @foobar # %bb.0: ret - .globl _baz -_baz: - calll _foobar - retl - .data .globl _rust_crate # @rust_crate .align 4 @@ -39,15 +39,6 @@ _rust_crate: .long _foobar-_rust_crate .long _foobar-_rust_crate -// Even though _baz and _foobar are in the same .text section, we keep the -// relocation for compatibility with the VC linker's /guard:cf and /incremental -// flags, even on mingw. - -// CHECK: Name: .text -// CHECK: Relocations [ -// CHECK-NEXT: 0x12 IMAGE_REL_I386_REL32 _foobar -// CHECK-NEXT: ] - // CHECK: Name: .data // CHECK: Relocations [ // CHECK-NEXT: 0x4 IMAGE_REL_I386_DIR32 _foobar diff --git a/llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll b/llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll new file mode 100644 index 00000000000000..50f222a2b7431d --- /dev/null +++ b/llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -opaque-pointers -atomic-expand -S -mtriple=powerpc64le-unknown-unknown \ +; RUN: %s | FileCheck %s +; RUN: opt -opaque-pointers -atomic-expand -S -mtriple=powerpc64-unknown-unknown \ +; RUN: %s | FileCheck %s + +define void @foo() #0 { +; CHECK-LABEL: @foo( +; CHECK-NEXT: bb0: +; CHECK-NEXT: br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[TMP0:%.*]] = load atomic ptr, ptr undef monotonic, align 8 +; CHECK-NEXT: call void @llvm.ppc.cfence.p0(ptr [[TMP0]]) +; CHECK-NEXT: unreachable +; CHECK: bb2: +; CHECK-NEXT: unreachable +; +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic {}*, {}** undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @bar() #0 { +; CHECK-LABEL: @bar( +; CHECK-NEXT: bb0: +; CHECK-NEXT: br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[TMP0:%.*]] = load atomic ptr, ptr undef monotonic, align 8 +; CHECK-NEXT: call void @llvm.ppc.cfence.p0(ptr [[TMP0]]) +; CHECK-NEXT: unreachable +; CHECK: bb2: +; CHECK-NEXT: unreachable +; +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic ptr, ptr undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @foobar() { +; CHECK-LABEL: @foobar( +; CHECK-NEXT: top: +; CHECK-NEXT: br i1 undef, label [[ERR86:%.*]], label [[PASS9:%.*]] +; CHECK: pass9: +; CHECK-NEXT: [[TMP0:%.*]] = load atomic ptr addrspace(10), ptr addrspace(11) undef monotonic, align 8 +; CHECK-NEXT: call void @llvm.ppc.cfence.p10(ptr addrspace(10) [[TMP0]]) +; CHECK-NEXT: unreachable +; CHECK: err86: +; CHECK-NEXT: unreachable +; +top: + br i1 undef, label %err86, label %pass9 + +pass9: ; preds = %top + %0 = load atomic {} addrspace(10)*, {} addrspace(10)* addrspace(11)* undef acquire, align 8 + unreachable + +err86: ; preds = %top + unreachable +} diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll new file mode 100644 index 00000000000000..39e50241c9cc6d --- /dev/null +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll @@ -0,0 +1,43 @@ +; RUN: opt -S -codegenprepare < %s | FileCheck %s + +target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" +target triple = "nvptx64-nvidia-cuda" + + +; ptrtoint/inttoptr combinations can introduce semantically-meaningful address space casts +; which we can't sink into an addrspacecast + +; CHECK-LABEL: @test +define void @test(i8* %input_ptr) { + ; CHECK-LABEL: l1: + ; CHECK-NOT: addrspacecast + %intptr = ptrtoint i8* %input_ptr to i64 + %ptr = inttoptr i64 %intptr to i32 addrspace(3)* + + br label %l1 +l1: + + store atomic i32 1, i32 addrspace(3)* %ptr unordered, align 4 + ret void +} + + +; we still should be able to look through multiple sequences of inttoptr/ptrtoint + +; CHECK-LABEL: @test2 +define void @test2(i8* %input_ptr) { + ; CHECK-LABEL: l2: + ; CHECK: bitcast + ; CHECK-NEXT: store + %intptr = ptrtoint i8* %input_ptr to i64 + %ptr = inttoptr i64 %intptr to i32 addrspace(3)* + + %intptr2 = ptrtoint i32 addrspace(3)* %ptr to i64 + %ptr2 = inttoptr i64 %intptr2 to i32* + + br label %l2 +l2: + + store atomic i32 1, i32* %ptr2 unordered, align 4 + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll new file mode 100644 index 00000000000000..5419efd454c736 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll @@ -0,0 +1,155 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -loop-vectorize < %s -mcpu=skylake | FileCheck %s +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:10:11:12:13" +target triple = "x86_64-unknown-linux-gnu" + +@jlplt_ijl_alloc_array_1d_10294_got = external dso_local local_unnamed_addr global void ()* + +define {} addrspace(10)* @japi1_vect_42283({} addrspace(10)** nocapture readonly %0, i32 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @japi1_vect_42283( +; CHECK-NEXT: top: +; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1:%.*]] to i64 +; CHECK-NEXT: [[TMP3:%.*]] = load atomic {} addrspace(10)* ({} addrspace(10)*, i64)*, {} addrspace(10)* ({} addrspace(10)*, i64)** bitcast (void ()** @jlplt_ijl_alloc_array_1d_10294_got to {} addrspace(10)* ({} addrspace(10)*, i64)**) unordered, align 8 +; CHECK-NEXT: [[TMP4:%.*]] = tail call {} addrspace(10)* [[TMP3]]({} addrspace(10)* null, i64 0) +; CHECK-NEXT: [[TMP5:%.*]] = bitcast {} addrspace(10)** [[TMP0:%.*]] to { {} addrspace(10)*, i64 } addrspace(10)** +; CHECK-NEXT: [[TMP6:%.*]] = load { {} addrspace(10)*, i64 } addrspace(10)*, { {} addrspace(10)*, i64 } addrspace(10)** [[TMP5]], align 8, !tbaa [[TBAA0:![0-9]+]] +; CHECK-NEXT: [[TMP7:%.*]] = bitcast {} addrspace(10)* [[TMP4]] to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* +; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* [[TMP7]] to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* +; CHECK-NEXT: [[TMP9:%.*]] = load { {} addrspace(10)*, i64 } addrspace(13)*, { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* [[TMP8]], align 8, !tbaa [[TBAA5:![0-9]+]] +; CHECK-NEXT: [[TMP10:%.*]] = bitcast { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]] to i8 addrspace(13)* +; CHECK-NEXT: [[DOTELT:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* [[TMP6]], i64 0, i32 0 +; CHECK-NEXT: [[DOTUNPACK:%.*]] = load {} addrspace(10)*, {} addrspace(10)* addrspace(10)* [[DOTELT]], align 8, !tbaa [[TBAA8:![0-9]+]] +; CHECK-NEXT: [[DOTELT1:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* [[TMP6]], i64 0, i32 1 +; CHECK-NEXT: [[DOTUNPACK2:%.*]] = load i64, i64 addrspace(10)* [[DOTELT1]], align 8, !tbaa [[TBAA8]] +; CHECK-NEXT: [[TMP11:%.*]] = add nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP11]], 16 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] +; CHECK: vector.scevcheck: +; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]]) +; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0 +; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = sub i64 0, [[MUL_RESULT]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, i8 addrspace(13)* [[TMP10]], i64 [[MUL_RESULT]] +; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i8 addrspace(13)* [[TMP13]], [[TMP10]] +; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], i64 0, i32 1 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = bitcast i64 addrspace(13)* [[SCEVGEP]] to { {} addrspace(10)*, i64 } addrspace(13)* +; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]]) +; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0 +; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1 +; CHECK-NEXT: [[SCEVGEP15:%.*]] = bitcast { {} addrspace(10)*, i64 } addrspace(13)* [[SCEVGEP1]] to i8 addrspace(13)* +; CHECK-NEXT: [[TMP16:%.*]] = sub i64 0, [[MUL_RESULT3]] +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, i8 addrspace(13)* [[SCEVGEP15]], i64 [[MUL_RESULT3]] +; CHECK-NEXT: [[TMP18:%.*]] = icmp ult i8 addrspace(13)* [[TMP17]], [[SCEVGEP15]] +; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[MUL_OVERFLOW4]] +; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP15]], [[TMP19]] +; CHECK-NEXT: br i1 [[TMP20]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP11]], 16 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP11]], [[N_MOD_VF]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT9]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT11]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT13]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT15]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT18:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT17]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT20:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT19]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT21:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT22:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT21]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], +; CHECK-NEXT: [[STEP_ADD6:%.*]] = add <4 x i64> [[STEP_ADD]], +; CHECK-NEXT: [[STEP_ADD7:%.*]] = add <4 x i64> [[STEP_ADD6]], +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[VEC_IND]], i32 0 +; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD]], i32 0 +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD6]], i32 0 +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD7]], i32 0 +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP21]], i32 8, <4 x i1> ), !tbaa [[TBAA10:![0-9]+]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT10]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP22]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT12]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP23]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT14]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP24]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[VEC_IND]], i32 1 +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD]], i32 1 +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD6]], i32 1 +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD7]], i32 1 +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT16]], <4 x i64 addrspace(13)*> [[TMP25]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT18]], <4 x i64 addrspace(13)*> [[TMP26]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT20]], <4 x i64 addrspace(13)*> [[TMP27]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT22]], <4 x i64 addrspace(13)*> [[TMP28]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD7]], +; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP11]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[L44:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[TOP:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ] +; CHECK-NEXT: br label [[L26:%.*]] +; CHECK: L26: +; CHECK-NEXT: [[VALUE_PHI5:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[TMP30:%.*]], [[L26]] ] +; CHECK-NEXT: [[DOTREPACK:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], i64 [[VALUE_PHI5]], i32 0 +; CHECK-NEXT: store {} addrspace(10)* [[DOTUNPACK]], {} addrspace(10)* addrspace(13)* [[DOTREPACK]], align 8, !tbaa [[TBAA10]] +; CHECK-NEXT: [[DOTREPACK4:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], i64 [[VALUE_PHI5]], i32 1 +; CHECK-NEXT: store i64 [[DOTUNPACK2]], i64 addrspace(13)* [[DOTREPACK4]], align 8, !tbaa [[TBAA10]] +; CHECK-NEXT: [[TMP30]] = add i64 [[VALUE_PHI5]], 1 +; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[VALUE_PHI5]], [[TMP2]] +; CHECK-NEXT: br i1 [[DOTNOT]], label [[L44]], label [[L26]], !llvm.loop [[LOOP14:![0-9]+]] +; CHECK: L44: +; CHECK-NEXT: ret {} addrspace(10)* null +; +top: + %2 = sext i32 %1 to i64 + %3 = load atomic {} addrspace(10)* ({} addrspace(10)*, i64)*, {} addrspace(10)* ({} addrspace(10)*, i64)** bitcast (void ()** @jlplt_ijl_alloc_array_1d_10294_got to {} addrspace(10)* ({} addrspace(10)*, i64)**) unordered, align 8 + %4 = tail call {} addrspace(10)* %3({} addrspace(10)* null, i64 0) + %5 = bitcast {} addrspace(10)** %0 to { {} addrspace(10)*, i64 } addrspace(10)** + %6 = load { {} addrspace(10)*, i64 } addrspace(10)*, { {} addrspace(10)*, i64 } addrspace(10)** %5, align 8, !tbaa !0 + %7 = bitcast {} addrspace(10)* %4 to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* + %8 = addrspacecast { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* %7 to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* + %9 = load { {} addrspace(10)*, i64 } addrspace(13)*, { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* %8, align 8, !tbaa !5 + %.elt = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* %6, i64 0, i32 0 + %.unpack = load {} addrspace(10)*, {} addrspace(10)* addrspace(10)* %.elt, align 8, !tbaa !8 + %.elt1 = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* %6, i64 0, i32 1 + %.unpack2 = load i64, i64 addrspace(10)* %.elt1, align 8, !tbaa !8 + br label %L26 + +L26: ; preds = %L26, %top + %value_phi5 = phi i64 [ 0, %top ], [ %10, %L26 ] + %.repack = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* %9, i64 %value_phi5, i32 0 + store {} addrspace(10)* %.unpack, {} addrspace(10)* addrspace(13)* %.repack, align 8, !tbaa !10 + %.repack4 = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* %9, i64 %value_phi5, i32 1 + store i64 %.unpack2, i64 addrspace(13)* %.repack4, align 8, !tbaa !10 + %10 = add i64 %value_phi5, 1 + %.not = icmp eq i64 %value_phi5, %2 + br i1 %.not, label %L44, label %L26 + +L44: ; preds = %L26 + ret {} addrspace(10)* null +} + +attributes #0 = { "target-cpu"="skylake-avx512" "target-features"="+xsaves,+xsavec,+prfchw,+lzcnt,+sahf,+pku,+avx512vl,+avx512bw,+avx512cd,+clwb,+clflushopt,+adx,+avx512dq,+avx512f,+bmi2,+avx2,+bmi,+fsgsbase,+f16c,+avx,+xsave,+aes,+popcnt,+movbe,+sse4.2,+sse4.1,+cx16,+fma,+ssse3,+pclmul,+sse3,-rdrnd,-rtm,-rdseed,-avx512ifma,-avx512pf,-avx512er,-sha,-prefetchwt1,-avx512vbmi,-waitpkg,-avx512vbmi2,-shstk,-gfni,-vaes,-vpclmulqdq,-avx512vnni,-avx512bitalg,-avx512vpopcntdq,-rdpid,-cldemote,-movdiri,-movdir64b,-enqcmd,-avx512vp2intersect,-serialize,-tsxldtrk,-pconfig,-amx-bf16,-amx-tile,-amx-int8,-sse4a,-xop,-lwp,-fma4,-tbm,-mwaitx,-xsaveopt,-clzero,-wbnoinvd,-avx512bf16,-ptwrite,+sse2,+mmx,+fxsr,+64bit,+cx8" } +attributes #1 = { inaccessiblemem_or_argmemonly } +attributes #2 = { allocsize(1) } + +!0 = !{!1, !1, i64 0} +!1 = !{!"jtbaa_value", !2, i64 0} +!2 = !{!"jtbaa_data", !3, i64 0} +!3 = !{!"jtbaa", !4, i64 0} +!4 = !{!"jtbaa"} +!5 = !{!6, !6, i64 0} +!6 = !{!"jtbaa_arrayptr", !7, i64 0} +!7 = !{!"jtbaa_array", !3, i64 0} +!8 = !{!9, !9, i64 0} +!9 = !{!"jtbaa_immut", !1, i64 0} +!10 = !{!11, !11, i64 0} +!11 = !{!"jtbaa_arraybuf", !2, i64 0} diff --git a/llvm/test/Transforms/MergeICmps/addressspaces.ll b/llvm/test/Transforms/MergeICmps/addressspaces.ll new file mode 100644 index 00000000000000..9a74b4a5b2ca4b --- /dev/null +++ b/llvm/test/Transforms/MergeICmps/addressspaces.ll @@ -0,0 +1,67 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -mergeicmps -S | FileCheck %s + +source_filename = "==" +target datalayout = "e-m:e-i64:64-n32:64" +target triple = "powerpc64le-unknown-linux-gnu" + +define void @juliaAS([2 x [5 x i64]] addrspace(11)* nocapture nonnull readonly align 8 dereferenceable(80) %0, [2 x [5 x i64]] addrspace(11)* nocapture nonnull readonly align 8 dereferenceable(80) %1) { +; CHECK-LABEL: @juliaAS( +; CHECK-NEXT: top: +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP0:%.*]], i64 0, i64 1, i64 2 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP0]], i64 0, i64 1, i64 3 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP0]], i64 0, i64 1, i64 4 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP1:%.*]], i64 0, i64 1, i64 2 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP1]], i64 0, i64 1, i64 3 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP1]], i64 0, i64 1, i64 4 +; CHECK-NEXT: [[TMP8:%.*]] = load i64, i64 addrspace(11)* [[TMP2]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = load i64, i64 addrspace(11)* [[TMP5]], align 8 +; CHECK-NEXT: [[DOTNOT17:%.*]] = icmp eq i64 [[TMP8]], [[TMP9]] +; CHECK-NEXT: br i1 [[DOTNOT17]], label [[L70:%.*]], label [[L90:%.*]] +; CHECK: L70: +; CHECK-NEXT: [[TMP10:%.*]] = load i64, i64 addrspace(11)* [[TMP3]], align 8 +; CHECK-NEXT: [[TMP11:%.*]] = load i64, i64 addrspace(11)* [[TMP6]], align 8 +; CHECK-NEXT: [[DOTNOT18:%.*]] = icmp eq i64 [[TMP10]], [[TMP11]] +; CHECK-NEXT: br i1 [[DOTNOT18]], label [[L74:%.*]], label [[L90]] +; CHECK: L74: +; CHECK-NEXT: [[TMP12:%.*]] = load i64, i64 addrspace(11)* [[TMP4]], align 8 +; CHECK-NEXT: [[TMP13:%.*]] = load i64, i64 addrspace(11)* [[TMP7]], align 8 +; CHECK-NEXT: [[DOTNOT19:%.*]] = icmp eq i64 [[TMP12]], [[TMP13]] +; CHECK-NEXT: br label [[L90]] +; CHECK: L90: +; CHECK-NEXT: [[VALUE_PHI2_OFF0:%.*]] = phi i1 [ false, [[TOP:%.*]] ], [ [[DOTNOT19]], [[L74]] ], [ false, [[L70]] ] +; CHECK-NEXT: ret void +; +top: + %2 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %0, i64 0, i64 1, i64 2 + %3 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %0, i64 0, i64 1, i64 3 + %4 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %0, i64 0, i64 1, i64 4 + %5 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %1, i64 0, i64 1, i64 2 + %6 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %1, i64 0, i64 1, i64 3 + %7 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %1, i64 0, i64 1, i64 4 + %8 = load i64, i64 addrspace(11)* %2, align 8 + %9 = load i64, i64 addrspace(11)* %5, align 8 + %.not17 = icmp eq i64 %8, %9 + br i1 %.not17, label %L70, label %L90 + +L70: ; preds = %top + %10 = load i64, i64 addrspace(11)* %3, align 8 + %11 = load i64, i64 addrspace(11)* %6, align 8 + %.not18 = icmp eq i64 %10, %11 + br i1 %.not18, label %L74, label %L90 + +L74: ; preds = %L70 + %12 = load i64, i64 addrspace(11)* %4, align 8 + %13 = load i64, i64 addrspace(11)* %7, align 8 + %.not19 = icmp eq i64 %12, %13 + br label %L90 + +L90: ; preds = %L74, %L70, %top + %value_phi2.off0 = phi i1 [ false, %top ], [ %.not19, %L74 ], [ false, %L70 ] + ret void +} + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"Debug Info Version", i32 3} +