From d0d6cfa7d17015e14b3d2afb4f8cc8af072db767 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 1 Oct 2021 19:45:59 -0400 Subject: [PATCH 01/34] Revert "[MC] Always emit relocations for same-section function references" This reverts commit 5a5ac65768d124d98a10e8520363a0a4be3f4e38. (cherry picked from commit ae2638d84b63af89ece7e30f39d435013ce42ee2) (cherry picked from commit 05848b6d4d8ccc212f3ba9d9f58af42f26983e2c) --- llvm/lib/MC/WinCOFFObjectWriter.cpp | 12 +++++------- llvm/test/MC/COFF/diff.s | 25 ++++++++----------------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp index aba2ad31553560..35f1ba3364594e 100644 --- a/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -686,14 +686,12 @@ void WinCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm, bool WinCOFFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl( const MCAssembler &Asm, const MCSymbol &SymA, const MCFragment &FB, bool InSet, bool IsPCRel) const { - // Don't drop relocations between functions, even if they are in the same text - // section. Multiple Visual C++ linker features depend on having the - // relocations present. The /INCREMENTAL flag will cause these relocations to - // point to thunks, and the /GUARD:CF flag assumes that it can use relocations - // to approximate the set of all address taken functions. LLD's implementation - // of /GUARD:CF also relies on the existance of these relocations. + // MS LINK expects to be able to replace all references to a function with a + // thunk to implement their /INCREMENTAL feature. Make sure we don't optimize + // away any relocations to functions. uint16_t Type = cast(SymA).getType(); - if ((Type >> COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION) + if (Asm.isIncrementalLinkerCompatible() && + (Type >> COFF::SCT_COMPLEX_TYPE_SHIFT) == COFF::IMAGE_SYM_DTYPE_FUNCTION) return false; return MCObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(Asm, SymA, FB, InSet, IsPCRel); diff --git a/llvm/test/MC/COFF/diff.s b/llvm/test/MC/COFF/diff.s index 90466b59d02522..640bf8189e0395 100644 --- a/llvm/test/MC/COFF/diff.s +++ b/llvm/test/MC/COFF/diff.s @@ -1,14 +1,19 @@ // RUN: llvm-mc -filetype=obj -triple i686-pc-mingw32 %s | llvm-readobj -S --sr --sd - | FileCheck %s -// COFF resolves differences between labels in the same section, unless that -// label is declared with function type. - .section baz, "xr" + .def X + .scl 2; + .type 32; + .endef .globl X X: mov Y-X+42, %eax retl + .def Y + .scl 2; + .type 32; + .endef .globl Y Y: retl @@ -25,11 +30,6 @@ _foobar: # @foobar # %bb.0: ret - .globl _baz -_baz: - calll _foobar - retl - .data .globl _rust_crate # @rust_crate .align 4 @@ -39,15 +39,6 @@ _rust_crate: .long _foobar-_rust_crate .long _foobar-_rust_crate -// Even though _baz and _foobar are in the same .text section, we keep the -// relocation for compatibility with the VC linker's /guard:cf and /incremental -// flags, even on mingw. - -// CHECK: Name: .text -// CHECK: Relocations [ -// CHECK-NEXT: 0x12 IMAGE_REL_I386_REL32 _foobar -// CHECK-NEXT: ] - // CHECK: Name: .data // CHECK: Relocations [ // CHECK-NEXT: 0x4 IMAGE_REL_I386_DIR32 _foobar From 0f3a27e505026c615c3117bed3bfaad27aa15a98 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 May 2018 11:56:55 -0400 Subject: [PATCH 02/34] Allow for custom address spaces Julia uses addressspaces for GC and we want these to be sanitized as well. (cherry picked from commit 3f53397f402b67341afe2bcb3a3316606b47d15c) (cherry picked from commit 58df73b7d510d59462d56092595cf9c91404c601) --- llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp index 180012198c42cf..54eac4d857f15f 100644 --- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp @@ -430,7 +430,9 @@ static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { // with them. if (Addr) { Type *PtrTy = cast(Addr->getType()->getScalarType()); - if (PtrTy->getPointerAddressSpace() != 0) + auto AS = PtrTy->getPointerAddressSpace(); + // Allow for custom addresspaces + if (AS != 0 && AS < 10) return false; } From a78d0b0be269c2d9464427ffade887c1cf4344ca Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 29 Sep 2021 15:17:47 -0400 Subject: [PATCH 03/34] [clang/CMake] Respect LLVM_TOOLS_INSTALL_DIR Otherwise clang installs all of its tools into `bin/` while LLVM installs its tools into (LLVM_TOOLS_INSTALL_DIR). I could swear this used to work (and in fact the julia build system assumes it), but I can't pin down a specific commit that would have broken this, and julia has been relying on pre-compiled binaries for a while now (that don't use this setting), so it may have been broken for quite a while. Differential Revision: https://reviews.llvm.org/D88630 (cherry picked from commit 6104e14b830c31dffb1b6bce1c6f9a0760993ff1) (cherry picked from commit f252e1795b885bf83f76ff4b029b0484aefebb31) (cherry picked from commit 9039ce8ab323e8e0bea24323a7231e4c53070def) --- clang/cmake/modules/AddClang.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/cmake/modules/AddClang.cmake b/clang/cmake/modules/AddClang.cmake index 9bbbfc032b7df9..6a3e9dbf4c2bbe 100644 --- a/clang/cmake/modules/AddClang.cmake +++ b/clang/cmake/modules/AddClang.cmake @@ -160,7 +160,7 @@ macro(add_clang_tool name) get_target_export_arg(${name} Clang export_to_clangtargets) install(TARGETS ${name} ${export_to_clangtargets} - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR} COMPONENT ${name}) if(NOT LLVM_ENABLE_IDE) From e68ee3f2b823528fa9a28b1ba4c6cc1912a249ba Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 29 Sep 2021 15:20:09 -0400 Subject: [PATCH 04/34] Don't sink ptrtoint/inttoptr sequences into non-noop addrspacecasts. Differential Revision: https://reviews.llvm.org/D92210 (cherry picked from commit 822b19a512f6939dca7f26c1fea1c847b93d25c0) (cherry picked from commit fcdf8f28a963931efce9095f92c5a2eb2fe88cb3) (cherry picked from commit 2208d33657bbe8b1e25ad3ca580ee7c1c01a6d13) --- llvm/lib/CodeGen/CodeGenPrepare.cpp | 32 +++++++++++--- .../NVPTX/dont-introduce-addrspacecast.ll | 43 +++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index c888adeafca5d1..328412821f20de 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -5205,18 +5205,29 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, WeakTrackingVH SunkAddrVH = SunkAddrs[Addr]; Value * SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr; + Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); if (SunkAddr) { LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for " << *MemoryInst << "\n"); - if (SunkAddr->getType() != Addr->getType()) - SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + if (SunkAddr->getType() != Addr->getType()) { + // Even though we only considered no-op addrspacecasts, + // semantically-meaningful conversions may still be present due to + // ptrtoint/inttoptr sequences. + if (SunkAddr->getType()->getPointerAddressSpace() != + Addr->getType()->getPointerAddressSpace() && + !DL->isNonIntegralPointerType(Addr->getType())) { + SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr"); + SunkAddr = + Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr"); + } else + SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + } } else if (AddrSinkUsingGEPs || (!AddrSinkUsingGEPs.getNumOccurrences() && SubtargetInfo->addrSinkUsingGEPs())) { // By default, we use the GEP-based method when AA is used later. This // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities. LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for " << *MemoryInst << "\n"); - Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); Value *ResultPtr = nullptr, *ResultIndex = nullptr; // First, find the pointer. @@ -5345,8 +5356,19 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, : Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); } - if (SunkAddr->getType() != Addr->getType()) - SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + if (SunkAddr->getType() != Addr->getType()) { + // Even though we only considered no-op addrspacecasts, + // semantically-meaningful conversions may still be present due to + // ptrtoint/inttoptr sequences. + if (SunkAddr->getType()->getPointerAddressSpace() != + Addr->getType()->getPointerAddressSpace() && + !DL->isNonIntegralPointerType(Addr->getType())) { + SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr"); + SunkAddr = + Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr"); + } else + SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); + } } } else { // We'd require a ptrtoint/inttoptr down the line, which we can't do for diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll new file mode 100644 index 00000000000000..39e50241c9cc6d --- /dev/null +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll @@ -0,0 +1,43 @@ +; RUN: opt -S -codegenprepare < %s | FileCheck %s + +target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" +target triple = "nvptx64-nvidia-cuda" + + +; ptrtoint/inttoptr combinations can introduce semantically-meaningful address space casts +; which we can't sink into an addrspacecast + +; CHECK-LABEL: @test +define void @test(i8* %input_ptr) { + ; CHECK-LABEL: l1: + ; CHECK-NOT: addrspacecast + %intptr = ptrtoint i8* %input_ptr to i64 + %ptr = inttoptr i64 %intptr to i32 addrspace(3)* + + br label %l1 +l1: + + store atomic i32 1, i32 addrspace(3)* %ptr unordered, align 4 + ret void +} + + +; we still should be able to look through multiple sequences of inttoptr/ptrtoint + +; CHECK-LABEL: @test2 +define void @test2(i8* %input_ptr) { + ; CHECK-LABEL: l2: + ; CHECK: bitcast + ; CHECK-NEXT: store + %intptr = ptrtoint i8* %input_ptr to i64 + %ptr = inttoptr i64 %intptr to i32 addrspace(3)* + + %intptr2 = ptrtoint i32 addrspace(3)* %ptr to i64 + %ptr2 = inttoptr i64 %intptr2 to i32* + + br label %l2 +l2: + + store atomic i32 1, i32* %ptr2 unordered, align 4 + ret void +} From b69564ac1f03e9f768a38ce70ddd2dbcde72b4bc Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 16 Jan 2021 17:36:09 -0500 Subject: [PATCH 05/34] Don't merge icmps derived from pointers with addressspaces IIUC we can't emit `memcmp` between pointers in addressspaces, doing so will trigger an assertion since the signature of the memcmp will not match it's arguments (https://bugs.llvm.org/show_bug.cgi?id=48661). This PR disables the attempt to merge icmps, when the pointer is in an addressspace. Differential Revision: https://reviews.llvm.org/D94813 (cherry picked from commit 458b259600f7efd82387eb7c4e09bdcee328106b) (cherry picked from commit aaf2d2763f878f73770ccfdaf40f77a565b24a73) --- .../Transforms/MergeICmps/addressspaces.ll | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 llvm/test/Transforms/MergeICmps/addressspaces.ll diff --git a/llvm/test/Transforms/MergeICmps/addressspaces.ll b/llvm/test/Transforms/MergeICmps/addressspaces.ll new file mode 100644 index 00000000000000..9a74b4a5b2ca4b --- /dev/null +++ b/llvm/test/Transforms/MergeICmps/addressspaces.ll @@ -0,0 +1,67 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -mergeicmps -S | FileCheck %s + +source_filename = "==" +target datalayout = "e-m:e-i64:64-n32:64" +target triple = "powerpc64le-unknown-linux-gnu" + +define void @juliaAS([2 x [5 x i64]] addrspace(11)* nocapture nonnull readonly align 8 dereferenceable(80) %0, [2 x [5 x i64]] addrspace(11)* nocapture nonnull readonly align 8 dereferenceable(80) %1) { +; CHECK-LABEL: @juliaAS( +; CHECK-NEXT: top: +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP0:%.*]], i64 0, i64 1, i64 2 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP0]], i64 0, i64 1, i64 3 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP0]], i64 0, i64 1, i64 4 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP1:%.*]], i64 0, i64 1, i64 2 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP1]], i64 0, i64 1, i64 3 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* [[TMP1]], i64 0, i64 1, i64 4 +; CHECK-NEXT: [[TMP8:%.*]] = load i64, i64 addrspace(11)* [[TMP2]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = load i64, i64 addrspace(11)* [[TMP5]], align 8 +; CHECK-NEXT: [[DOTNOT17:%.*]] = icmp eq i64 [[TMP8]], [[TMP9]] +; CHECK-NEXT: br i1 [[DOTNOT17]], label [[L70:%.*]], label [[L90:%.*]] +; CHECK: L70: +; CHECK-NEXT: [[TMP10:%.*]] = load i64, i64 addrspace(11)* [[TMP3]], align 8 +; CHECK-NEXT: [[TMP11:%.*]] = load i64, i64 addrspace(11)* [[TMP6]], align 8 +; CHECK-NEXT: [[DOTNOT18:%.*]] = icmp eq i64 [[TMP10]], [[TMP11]] +; CHECK-NEXT: br i1 [[DOTNOT18]], label [[L74:%.*]], label [[L90]] +; CHECK: L74: +; CHECK-NEXT: [[TMP12:%.*]] = load i64, i64 addrspace(11)* [[TMP4]], align 8 +; CHECK-NEXT: [[TMP13:%.*]] = load i64, i64 addrspace(11)* [[TMP7]], align 8 +; CHECK-NEXT: [[DOTNOT19:%.*]] = icmp eq i64 [[TMP12]], [[TMP13]] +; CHECK-NEXT: br label [[L90]] +; CHECK: L90: +; CHECK-NEXT: [[VALUE_PHI2_OFF0:%.*]] = phi i1 [ false, [[TOP:%.*]] ], [ [[DOTNOT19]], [[L74]] ], [ false, [[L70]] ] +; CHECK-NEXT: ret void +; +top: + %2 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %0, i64 0, i64 1, i64 2 + %3 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %0, i64 0, i64 1, i64 3 + %4 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %0, i64 0, i64 1, i64 4 + %5 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %1, i64 0, i64 1, i64 2 + %6 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %1, i64 0, i64 1, i64 3 + %7 = getelementptr inbounds [2 x [5 x i64]], [2 x [5 x i64]] addrspace(11)* %1, i64 0, i64 1, i64 4 + %8 = load i64, i64 addrspace(11)* %2, align 8 + %9 = load i64, i64 addrspace(11)* %5, align 8 + %.not17 = icmp eq i64 %8, %9 + br i1 %.not17, label %L70, label %L90 + +L70: ; preds = %top + %10 = load i64, i64 addrspace(11)* %3, align 8 + %11 = load i64, i64 addrspace(11)* %6, align 8 + %.not18 = icmp eq i64 %10, %11 + br i1 %.not18, label %L74, label %L90 + +L74: ; preds = %L70 + %12 = load i64, i64 addrspace(11)* %4, align 8 + %13 = load i64, i64 addrspace(11)* %7, align 8 + %.not19 = icmp eq i64 %12, %13 + br label %L90 + +L90: ; preds = %L74, %L70, %top + %value_phi2.off0 = phi i1 [ false, %top ], [ %.not19, %L74 ], [ false, %L70 ] + ret void +} + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"Debug Info Version", i32 3} + From ca588e8768eb27a7dde2892b0a2dc9f645800273 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 1 Mar 2021 16:42:05 -0500 Subject: [PATCH 06/34] AArch64: Remove Bad optimization Removes the code responsible for causing https://bugs.llvm.org/show_bug.cgi?id=49357. A fix is in progress upstream, but I don't think it's easy, so this fixes the bug in the meantime. The optimization it does is minor. (cherry picked from commit e4f1085c5c04e106e5a7ee72c9f4dfe1dccb7b94) (cherry picked from commit 7627d618e0e62cdb6d9394d37198dab2a02252f1) --- llvm/lib/Target/AArch64/AArch64FastISel.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index dc5e6807945d72..a80c08b94948ec 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -4504,23 +4504,6 @@ bool AArch64FastISel::selectIntExt(const Instruction *I) { // Try to optimize already sign-/zero-extended values from function arguments. bool IsZExt = isa(I); - if (const auto *Arg = dyn_cast(I->getOperand(0))) { - if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) { - if (RetVT == MVT::i64 && SrcVT != MVT::i64) { - Register ResultReg = createResultReg(&AArch64::GPR64RegClass); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, - TII.get(AArch64::SUBREG_TO_REG), ResultReg) - .addImm(0) - .addReg(SrcReg) - .addImm(AArch64::sub_32); - SrcReg = ResultReg; - } - - updateValueMap(I, SrcReg); - return true; - } - } - unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt); if (!ResultReg) return false; From 503f510ed231723fbdd510677c6d68bd199a29e8 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Sun, 3 Apr 2022 20:09:03 -0700 Subject: [PATCH 07/34] [LV] Handle non-integral types when considering interleave widening legality In general, anywhere we might need to insert a blind bitcast, we need to make sure the types are losslessly convertible. This fixes pr54634. (cherry picked from commit 88de27e3fd9fccec9abd1d224282a6374931fb64) --- .../Transforms/Vectorize/LoopVectorize.cpp | 21 +++ .../Transforms/LoopVectorize/X86/pr54634.ll | 155 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr54634.ll diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 46ff0994e04e7f..e629598ad36b7d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4892,6 +4892,27 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened( if (hasIrregularType(ScalarTy, DL)) return false; + // If the group involves a non-integral pointer, we may not be able to + // losslessly cast all values to a common type. + unsigned InterleaveFactor = Group->getFactor(); + bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy); + for (unsigned i = 0; i < InterleaveFactor; i++) { + Instruction *Member = Group->getMember(i); + if (!Member) + continue; + auto *MemberTy = getLoadStoreType(Member); + bool MemberNI = DL.isNonIntegralPointerType(MemberTy); + // Don't coerce non-integral pointers to integers or vice versa. + if (MemberNI != ScalarNI) { + // TODO: Consider adding special nullptr value case here + return false; + } else if (MemberNI && ScalarNI && + ScalarTy->getPointerAddressSpace() != + MemberTy->getPointerAddressSpace()) { + return false; + } + } + // Check if masking is required. // A Group may need masking for one of two reasons: it resides in a block that // needs predication, or it was decided to use masking to deal with gaps diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll new file mode 100644 index 00000000000000..5419efd454c736 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll @@ -0,0 +1,155 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -loop-vectorize < %s -mcpu=skylake | FileCheck %s +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:10:11:12:13" +target triple = "x86_64-unknown-linux-gnu" + +@jlplt_ijl_alloc_array_1d_10294_got = external dso_local local_unnamed_addr global void ()* + +define {} addrspace(10)* @japi1_vect_42283({} addrspace(10)** nocapture readonly %0, i32 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @japi1_vect_42283( +; CHECK-NEXT: top: +; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1:%.*]] to i64 +; CHECK-NEXT: [[TMP3:%.*]] = load atomic {} addrspace(10)* ({} addrspace(10)*, i64)*, {} addrspace(10)* ({} addrspace(10)*, i64)** bitcast (void ()** @jlplt_ijl_alloc_array_1d_10294_got to {} addrspace(10)* ({} addrspace(10)*, i64)**) unordered, align 8 +; CHECK-NEXT: [[TMP4:%.*]] = tail call {} addrspace(10)* [[TMP3]]({} addrspace(10)* null, i64 0) +; CHECK-NEXT: [[TMP5:%.*]] = bitcast {} addrspace(10)** [[TMP0:%.*]] to { {} addrspace(10)*, i64 } addrspace(10)** +; CHECK-NEXT: [[TMP6:%.*]] = load { {} addrspace(10)*, i64 } addrspace(10)*, { {} addrspace(10)*, i64 } addrspace(10)** [[TMP5]], align 8, !tbaa [[TBAA0:![0-9]+]] +; CHECK-NEXT: [[TMP7:%.*]] = bitcast {} addrspace(10)* [[TMP4]] to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* +; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* [[TMP7]] to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* +; CHECK-NEXT: [[TMP9:%.*]] = load { {} addrspace(10)*, i64 } addrspace(13)*, { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* [[TMP8]], align 8, !tbaa [[TBAA5:![0-9]+]] +; CHECK-NEXT: [[TMP10:%.*]] = bitcast { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]] to i8 addrspace(13)* +; CHECK-NEXT: [[DOTELT:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* [[TMP6]], i64 0, i32 0 +; CHECK-NEXT: [[DOTUNPACK:%.*]] = load {} addrspace(10)*, {} addrspace(10)* addrspace(10)* [[DOTELT]], align 8, !tbaa [[TBAA8:![0-9]+]] +; CHECK-NEXT: [[DOTELT1:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* [[TMP6]], i64 0, i32 1 +; CHECK-NEXT: [[DOTUNPACK2:%.*]] = load i64, i64 addrspace(10)* [[DOTELT1]], align 8, !tbaa [[TBAA8]] +; CHECK-NEXT: [[TMP11:%.*]] = add nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP11]], 16 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] +; CHECK: vector.scevcheck: +; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]]) +; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0 +; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = sub i64 0, [[MUL_RESULT]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, i8 addrspace(13)* [[TMP10]], i64 [[MUL_RESULT]] +; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i8 addrspace(13)* [[TMP13]], [[TMP10]] +; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], i64 0, i32 1 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = bitcast i64 addrspace(13)* [[SCEVGEP]] to { {} addrspace(10)*, i64 } addrspace(13)* +; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]]) +; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0 +; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1 +; CHECK-NEXT: [[SCEVGEP15:%.*]] = bitcast { {} addrspace(10)*, i64 } addrspace(13)* [[SCEVGEP1]] to i8 addrspace(13)* +; CHECK-NEXT: [[TMP16:%.*]] = sub i64 0, [[MUL_RESULT3]] +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, i8 addrspace(13)* [[SCEVGEP15]], i64 [[MUL_RESULT3]] +; CHECK-NEXT: [[TMP18:%.*]] = icmp ult i8 addrspace(13)* [[TMP17]], [[SCEVGEP15]] +; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[MUL_OVERFLOW4]] +; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP15]], [[TMP19]] +; CHECK-NEXT: br i1 [[TMP20]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP11]], 16 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP11]], [[N_MOD_VF]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT9]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT11]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x {} addrspace(10)*> poison, {} addrspace(10)* [[DOTUNPACK]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x {} addrspace(10)*> [[BROADCAST_SPLATINSERT13]], <4 x {} addrspace(10)*> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT15]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT18:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT17]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT20:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT19]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT21:%.*]] = insertelement <4 x i64> poison, i64 [[DOTUNPACK2]], i32 0 +; CHECK-NEXT: [[BROADCAST_SPLAT22:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT21]], <4 x i64> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], +; CHECK-NEXT: [[STEP_ADD6:%.*]] = add <4 x i64> [[STEP_ADD]], +; CHECK-NEXT: [[STEP_ADD7:%.*]] = add <4 x i64> [[STEP_ADD6]], +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[VEC_IND]], i32 0 +; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD]], i32 0 +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD6]], i32 0 +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD7]], i32 0 +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP21]], i32 8, <4 x i1> ), !tbaa [[TBAA10:![0-9]+]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT10]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP22]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT12]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP23]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4p10sl_s.v4p13p10sl_s(<4 x {} addrspace(10)*> [[BROADCAST_SPLAT14]], <4 x {} addrspace(10)* addrspace(13)*> [[TMP24]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[VEC_IND]], i32 1 +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD]], i32 1 +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD6]], i32 1 +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], <4 x i64> [[STEP_ADD7]], i32 1 +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT16]], <4 x i64 addrspace(13)*> [[TMP25]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT18]], <4 x i64 addrspace(13)*> [[TMP26]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT20]], <4 x i64 addrspace(13)*> [[TMP27]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p13i64(<4 x i64> [[BROADCAST_SPLAT22]], <4 x i64 addrspace(13)*> [[TMP28]], i32 8, <4 x i1> ), !tbaa [[TBAA10]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD7]], +; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP11]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[L44:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[TOP:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ] +; CHECK-NEXT: br label [[L26:%.*]] +; CHECK: L26: +; CHECK-NEXT: [[VALUE_PHI5:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[TMP30:%.*]], [[L26]] ] +; CHECK-NEXT: [[DOTREPACK:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], i64 [[VALUE_PHI5]], i32 0 +; CHECK-NEXT: store {} addrspace(10)* [[DOTUNPACK]], {} addrspace(10)* addrspace(13)* [[DOTREPACK]], align 8, !tbaa [[TBAA10]] +; CHECK-NEXT: [[DOTREPACK4:%.*]] = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* [[TMP9]], i64 [[VALUE_PHI5]], i32 1 +; CHECK-NEXT: store i64 [[DOTUNPACK2]], i64 addrspace(13)* [[DOTREPACK4]], align 8, !tbaa [[TBAA10]] +; CHECK-NEXT: [[TMP30]] = add i64 [[VALUE_PHI5]], 1 +; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[VALUE_PHI5]], [[TMP2]] +; CHECK-NEXT: br i1 [[DOTNOT]], label [[L44]], label [[L26]], !llvm.loop [[LOOP14:![0-9]+]] +; CHECK: L44: +; CHECK-NEXT: ret {} addrspace(10)* null +; +top: + %2 = sext i32 %1 to i64 + %3 = load atomic {} addrspace(10)* ({} addrspace(10)*, i64)*, {} addrspace(10)* ({} addrspace(10)*, i64)** bitcast (void ()** @jlplt_ijl_alloc_array_1d_10294_got to {} addrspace(10)* ({} addrspace(10)*, i64)**) unordered, align 8 + %4 = tail call {} addrspace(10)* %3({} addrspace(10)* null, i64 0) + %5 = bitcast {} addrspace(10)** %0 to { {} addrspace(10)*, i64 } addrspace(10)** + %6 = load { {} addrspace(10)*, i64 } addrspace(10)*, { {} addrspace(10)*, i64 } addrspace(10)** %5, align 8, !tbaa !0 + %7 = bitcast {} addrspace(10)* %4 to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* + %8 = addrspacecast { {} addrspace(10)*, i64 } addrspace(13)* addrspace(10)* %7 to { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* + %9 = load { {} addrspace(10)*, i64 } addrspace(13)*, { {} addrspace(10)*, i64 } addrspace(13)* addrspace(11)* %8, align 8, !tbaa !5 + %.elt = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* %6, i64 0, i32 0 + %.unpack = load {} addrspace(10)*, {} addrspace(10)* addrspace(10)* %.elt, align 8, !tbaa !8 + %.elt1 = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(10)* %6, i64 0, i32 1 + %.unpack2 = load i64, i64 addrspace(10)* %.elt1, align 8, !tbaa !8 + br label %L26 + +L26: ; preds = %L26, %top + %value_phi5 = phi i64 [ 0, %top ], [ %10, %L26 ] + %.repack = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* %9, i64 %value_phi5, i32 0 + store {} addrspace(10)* %.unpack, {} addrspace(10)* addrspace(13)* %.repack, align 8, !tbaa !10 + %.repack4 = getelementptr inbounds { {} addrspace(10)*, i64 }, { {} addrspace(10)*, i64 } addrspace(13)* %9, i64 %value_phi5, i32 1 + store i64 %.unpack2, i64 addrspace(13)* %.repack4, align 8, !tbaa !10 + %10 = add i64 %value_phi5, 1 + %.not = icmp eq i64 %value_phi5, %2 + br i1 %.not, label %L44, label %L26 + +L44: ; preds = %L26 + ret {} addrspace(10)* null +} + +attributes #0 = { "target-cpu"="skylake-avx512" "target-features"="+xsaves,+xsavec,+prfchw,+lzcnt,+sahf,+pku,+avx512vl,+avx512bw,+avx512cd,+clwb,+clflushopt,+adx,+avx512dq,+avx512f,+bmi2,+avx2,+bmi,+fsgsbase,+f16c,+avx,+xsave,+aes,+popcnt,+movbe,+sse4.2,+sse4.1,+cx16,+fma,+ssse3,+pclmul,+sse3,-rdrnd,-rtm,-rdseed,-avx512ifma,-avx512pf,-avx512er,-sha,-prefetchwt1,-avx512vbmi,-waitpkg,-avx512vbmi2,-shstk,-gfni,-vaes,-vpclmulqdq,-avx512vnni,-avx512bitalg,-avx512vpopcntdq,-rdpid,-cldemote,-movdiri,-movdir64b,-enqcmd,-avx512vp2intersect,-serialize,-tsxldtrk,-pconfig,-amx-bf16,-amx-tile,-amx-int8,-sse4a,-xop,-lwp,-fma4,-tbm,-mwaitx,-xsaveopt,-clzero,-wbnoinvd,-avx512bf16,-ptwrite,+sse2,+mmx,+fxsr,+64bit,+cx8" } +attributes #1 = { inaccessiblemem_or_argmemonly } +attributes #2 = { allocsize(1) } + +!0 = !{!1, !1, i64 0} +!1 = !{!"jtbaa_value", !2, i64 0} +!2 = !{!"jtbaa_data", !3, i64 0} +!3 = !{!"jtbaa", !4, i64 0} +!4 = !{!"jtbaa"} +!5 = !{!6, !6, i64 0} +!6 = !{!"jtbaa_arrayptr", !7, i64 0} +!7 = !{!"jtbaa_array", !3, i64 0} +!8 = !{!9, !9, i64 0} +!9 = !{!"jtbaa_immut", !1, i64 0} +!10 = !{!11, !11, i64 0} +!11 = !{!"jtbaa_arraybuf", !2, i64 0} From 02db4d1880804c8a58a0a666a3e97f8014d50172 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 30 Apr 2022 19:00:11 +0100 Subject: [PATCH 08/34] Add support for unwinding during prologue/epilogue (cherry picked from commit 5393efbd8a4c7555b9f9fdf185c486c6b05f0c19) --- libunwind/src/CompactUnwinder.hpp | 156 ++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/libunwind/src/CompactUnwinder.hpp b/libunwind/src/CompactUnwinder.hpp index 0b2b5e111bfc26..ad0e042cb6ba11 100644 --- a/libunwind/src/CompactUnwinder.hpp +++ b/libunwind/src/CompactUnwinder.hpp @@ -310,6 +310,50 @@ int CompactUnwinder_x86_64::stepWithCompactEncodingRBPFrame( uint32_t savedRegistersLocations = EXTRACT_BITS(compactEncoding, UNWIND_X86_64_RBP_FRAME_REGISTERS); + // If we have not stored EBP yet + if (functionStart == registers.getIP()) { + uint64_t rsp = registers.getSP(); + // old esp is ebp less return address + registers.setSP(rsp+8); + // pop return address into eip + registers.setIP(addressSpace.get64(rsp)); + + return UNW_STEP_SUCCESS; + } else if (functionStart + 1 == registers.getIP()) { + uint64_t rsp = registers.getSP(); + // old esp is ebp less return address + registers.setSP(rsp + 16); + // pop return address into eip + registers.setIP(addressSpace.get64(rsp + 8)); + + return UNW_STEP_SUCCESS; + } + + // If we're about to return, we've already popped the base pointer + uint8_t b = addressSpace.get8(registers.getIP()); + + // This is a hack to detect VZEROUPPER but in between popq rbp and ret + // It's not pretty but it works + if (b == 0xC5) { + if ((b = addressSpace.get8(registers.getIP() + 1)) == 0xF8 && + (b = addressSpace.get8(registers.getIP() + 2)) == 0x77) + b = addressSpace.get8(registers.getIP() + 3); + else + goto skip_ret; + } + + if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) { + uint64_t rbp = registers.getSP(); + // old esp is ebp less return address + registers.setSP(rbp + 16); + // pop return address into eip + registers.setIP(addressSpace.get64(rbp + 8)); + + return UNW_STEP_SUCCESS; + } + + skip_ret: + uint64_t savedRegisters = registers.getRBP() - 8 * savedRegistersOffset; for (int i = 0; i < 5; ++i) { switch (savedRegistersLocations & 0x7) { @@ -430,6 +474,118 @@ int CompactUnwinder_x86_64::stepWithCompactEncodingFrameless( } } } + + // Note that the order of these registers is so that + // registersSaved[0] is the one that will be pushed onto the stack last. + // Thus, if we want to walk this from the top, we need to go in reverse. + assert(regCount <= 6); + + // check whether we are still in the prologue + uint64_t curAddr = functionStart; + if (regCount > 0) { + for (int8_t i = (int8_t)(regCount) - 1; i >= 0; --i) { + if (registers.getIP() == curAddr) { + // None of the registers have been modified yet, so we don't need to reload them + framelessUnwind(addressSpace, registers.getSP() + 8 * (regCount - (uint64_t)(i + 1)), registers); + return UNW_STEP_SUCCESS; + } else { + assert(curAddr < registers.getIP()); + } + + + // pushq %rbp and pushq %rbx is 1 byte. Everything else 2 + if ((UNWIND_X86_64_REG_RBP == registersSaved[i]) || + (UNWIND_X86_64_REG_RBX == registersSaved[i])) + curAddr += 1; + else + curAddr += 2; + } + } + if (registers.getIP() == curAddr) { + // None of the registers have been modified yet, so we don't need to reload them + framelessUnwind(addressSpace, registers.getSP() + 8*regCount, registers); + return UNW_STEP_SUCCESS; + } else { + assert(curAddr < registers.getIP()); + } + + + // And now for the epilogue + { + uint8_t i = 0; + uint64_t p = registers.getIP(); + uint8_t b = 0; + + while (true) { + b = addressSpace.get8(p++); + // This is a hack to detect VZEROUPPER but in between the popq's and ret + // It's not pretty but it works + if (b == 0xC5) { + if ((b = addressSpace.get8(p++)) == 0xF8 && (b = addressSpace.get8(p++)) == 0x77) + b = addressSpace.get8(p++); + else + break; + } + // popq %rbx popq %rbp + if (b == 0x5B || b == 0x5D) { + i++; + } else if (b == 0x41) { + b = addressSpace.get8(p++); + if (b == 0x5C || b == 0x5D || b == 0x5E || b == 0x5F) + i++; + else + break; + } else if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) { + // i pop's haven't happened yet + uint64_t savedRegisters = registers.getSP() + 8 * i; + if (regCount > 0) { + for (int8_t j = (int8_t)(regCount) - 1; j >= (int8_t)(regCount) - i; --j) { + uint64_t addr = savedRegisters - 8 * (regCount - (uint64_t)(j)); + switch (registersSaved[j]) { + case UNWIND_X86_64_REG_RBX: + registers.setRBX(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R12: + registers.setR12(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R13: + registers.setR13(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R14: + registers.setR14(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_R15: + registers.setR15(addressSpace.get64(addr)); + break; + case UNWIND_X86_64_REG_RBP: + registers.setRBP(addressSpace.get64(addr)); + break; + default: + _LIBUNWIND_DEBUG_LOG("bad register for frameless, encoding=%08X for " + "function starting at 0x%llX", + encoding, functionStart); + _LIBUNWIND_ABORT("invalid compact unwind encoding"); + } + } + } + framelessUnwind(addressSpace, savedRegisters, registers); + return UNW_STEP_SUCCESS; + } else { + break; + } + } + } + + /* + 0x10fe2733a: 5b popq %rbx + 0x10fe2733b: 41 5c popq %r12 + 0x10fe2733d: 41 5d popq %r13 + 0x10fe2733f: 41 5e popq %r14 + 0x10fe27341: 41 5f popq %r15 + 0x10fe27343: 5d popq %rbp + */ + + uint64_t savedRegisters = registers.getSP() + stackSize - 8 - 8 * regCount; for (uint32_t i = 0; i < regCount; ++i) { switch (registersSaved[i]) { From 08d2371d7e5572ece919d719fafca205290bad57 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Tue, 18 Jan 2022 13:32:28 -0600 Subject: [PATCH 09/34] [LLD] Respect LLVM_TOOLS_INSTALL_DIR Co-authored-by: Valentin Churavy Co-authored-by: Julian P Samaroo (cherry picked from commit a0defe021cee2076dc161eceeaab70297b386b91) --- lld/cmake/modules/AddLLD.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lld/cmake/modules/AddLLD.cmake b/lld/cmake/modules/AddLLD.cmake index dd2898ce623647..5e176976b60300 100644 --- a/lld/cmake/modules/AddLLD.cmake +++ b/lld/cmake/modules/AddLLD.cmake @@ -20,7 +20,7 @@ macro(add_lld_library name) ${export_to_lldtargets} LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX} ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX} - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR}) if (${ARG_SHARED} AND NOT CMAKE_CONFIGURATION_TYPES) add_llvm_install_targets(install-${name} @@ -47,7 +47,7 @@ macro(add_lld_tool name) get_target_export_arg(${name} LLD export_to_lldtargets) install(TARGETS ${name} ${export_to_lldtargets} - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" + RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR} COMPONENT ${name}) if(NOT CMAKE_CONFIGURATION_TYPES) From 16f0adfc3013f828203cae59a1d2c58b65fe3249 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Mon, 2 May 2022 10:04:47 -0400 Subject: [PATCH 10/34] [Sanitizers] Guard FP_XSTATE_MAGIC1 usage by GLIBC version Follow-up on https://reviews.llvm.org/D118970 FP_XSTATE_MAGIC1 is only available on glibc 2.27 and upwards Differential Revision: https://reviews.llvm.org/D124770 --- .../lib/sanitizer_common/sanitizer_platform_limits_posix.cpp | 2 +- compiler-rt/test/msan/Linux/signal_mcontext.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp index 32b8f47ed6338e..aba3240c532bd5 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -217,7 +217,7 @@ namespace __sanitizer { unsigned struct_sockaddr_sz = sizeof(struct sockaddr); unsigned ucontext_t_sz(void *ctx) { -# if SANITIZER_GLIBC && SANITIZER_X64 +# if SANITIZER_GLIBC && SANITIZER_X64 && __GLIBC_PREREQ (2, 27) // See kernel arch/x86/kernel/fpu/signal.c for details. const auto *fpregs = static_cast(ctx)->uc_mcontext.fpregs; // The member names differ across header versions, but the actual layout diff --git a/compiler-rt/test/msan/Linux/signal_mcontext.cpp b/compiler-rt/test/msan/Linux/signal_mcontext.cpp index b49451fbb730b6..11ef74e7462bbe 100644 --- a/compiler-rt/test/msan/Linux/signal_mcontext.cpp +++ b/compiler-rt/test/msan/Linux/signal_mcontext.cpp @@ -10,7 +10,7 @@ void handler(int sig, siginfo_t *info, void *uctx) { __msan_check_mem_is_initialized(uctx, sizeof(ucontext_t)); -#if defined(__GLIBC__) && defined(__x86_64__) +#if defined(__GLIBC__) && defined(__x86_64__) && __GLIBC_PREREQ(2, 27) auto *mctx = &static_cast(uctx)->uc_mcontext; if (auto *fpregs = mctx->fpregs) { // The member names differ across header versions, but the actual layout From 3c1151c0f6c5b265ec2b3a176fe12be4b23252bf Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 17 Jun 2022 13:49:53 -0400 Subject: [PATCH 11/34] [PowerPC] Allow llvm.ppc.cfence to accept pointer types Backport https://reviews.llvm.org/D127554 (cherry picked from commit 2f4460bd46aa80d4fe0d80c3dabcb10379e8d61b) --- llvm/include/llvm/IR/IntrinsicsPowerPC.td | 6 +- llvm/test/CodeGen/PowerPC/issue55983.ll | 66 +++++++++++++++++ .../AtomicExpand/PowerPC/issue55983.ll | 71 +++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/PowerPC/issue55983.ll create mode 100644 llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td index b01fa10763b83e..6e5b9cbd9a4499 100644 --- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td +++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td @@ -1459,7 +1459,11 @@ def int_ppc_tsuspend : GCCBuiltin<"__builtin_tsuspend">, def int_ppc_ttest : GCCBuiltin<"__builtin_ttest">, Intrinsic<[llvm_i64_ty], [], []>; -def int_ppc_cfence : Intrinsic<[], [llvm_anyint_ty], []>; +// We currently use llvm.ppc.cfence in the context of atomic load which +// in LLVM IR requires its type to be one of integer, pointer and +// float point type. So llvm_any_ty here refers to type mentioned above. +// Backend is supposed to lower these types to appropriate MVTs. +def int_ppc_cfence : Intrinsic<[], [llvm_any_ty], []>; // PowerPC set FPSCR Intrinsic Definitions. def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">, diff --git a/llvm/test/CodeGen/PowerPC/issue55983.ll b/llvm/test/CodeGen/PowerPC/issue55983.ll new file mode 100644 index 00000000000000..7ebde001ada7f4 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/issue55983.ll @@ -0,0 +1,66 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -opaque-pointers -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s +; RUN: llc -opaque-pointers -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s + +define void @foo() #0 { +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %bb0 +; CHECK-NEXT: bc 12, 20, .LBB0_2 +; CHECK-NEXT: # %bb.1: # %bb1 +; CHECK-NEXT: ld 3, 0(3) +; CHECK-NEXT: cmpd 7, 3, 3 +; CHECK-NEXT: bne- 7, .+4 +; CHECK-NEXT: isync +; CHECK-NEXT: .LBB0_2: # %bb2 +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic {}*, {}** undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @bar() #0 { +; CHECK-LABEL: bar: +; CHECK: # %bb.0: # %bb0 +; CHECK-NEXT: bc 12, 20, .LBB1_2 +; CHECK-NEXT: # %bb.1: # %bb1 +; CHECK-NEXT: ld 3, 0(3) +; CHECK-NEXT: cmpd 7, 3, 3 +; CHECK-NEXT: bne- 7, .+4 +; CHECK-NEXT: isync +; CHECK-NEXT: .LBB1_2: # %bb2 +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic ptr, ptr undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @foobar() { +; CHECK-LABEL: foobar: +; CHECK: # %bb.0: # %top +; CHECK-NEXT: bc 4, 20, .LBB2_2 +; CHECK-NEXT: # %bb.1: # %err86 +; CHECK-NEXT: .LBB2_2: # %pass9 +; CHECK-NEXT: ld 3, 0(3) +; CHECK-NEXT: cmpd 7, 3, 3 +; CHECK-NEXT: bne- 7, .+4 +; CHECK-NEXT: isync +top: + br i1 undef, label %err86, label %pass9 + +pass9: ; preds = %top + %0 = load atomic {} addrspace(10)*, {} addrspace(10)* addrspace(11)* undef acquire, align 8 + unreachable + +err86: ; preds = %top + unreachable +} diff --git a/llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll b/llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll new file mode 100644 index 00000000000000..50f222a2b7431d --- /dev/null +++ b/llvm/test/Transforms/AtomicExpand/PowerPC/issue55983.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -opaque-pointers -atomic-expand -S -mtriple=powerpc64le-unknown-unknown \ +; RUN: %s | FileCheck %s +; RUN: opt -opaque-pointers -atomic-expand -S -mtriple=powerpc64-unknown-unknown \ +; RUN: %s | FileCheck %s + +define void @foo() #0 { +; CHECK-LABEL: @foo( +; CHECK-NEXT: bb0: +; CHECK-NEXT: br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[TMP0:%.*]] = load atomic ptr, ptr undef monotonic, align 8 +; CHECK-NEXT: call void @llvm.ppc.cfence.p0(ptr [[TMP0]]) +; CHECK-NEXT: unreachable +; CHECK: bb2: +; CHECK-NEXT: unreachable +; +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic {}*, {}** undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @bar() #0 { +; CHECK-LABEL: @bar( +; CHECK-NEXT: bb0: +; CHECK-NEXT: br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[TMP0:%.*]] = load atomic ptr, ptr undef monotonic, align 8 +; CHECK-NEXT: call void @llvm.ppc.cfence.p0(ptr [[TMP0]]) +; CHECK-NEXT: unreachable +; CHECK: bb2: +; CHECK-NEXT: unreachable +; +bb0: + br i1 undef, label %bb1, label %bb2 + +bb1: + %0 = load atomic ptr, ptr undef acquire, align 8 + unreachable + +bb2: + unreachable +} + +define void @foobar() { +; CHECK-LABEL: @foobar( +; CHECK-NEXT: top: +; CHECK-NEXT: br i1 undef, label [[ERR86:%.*]], label [[PASS9:%.*]] +; CHECK: pass9: +; CHECK-NEXT: [[TMP0:%.*]] = load atomic ptr addrspace(10), ptr addrspace(11) undef monotonic, align 8 +; CHECK-NEXT: call void @llvm.ppc.cfence.p10(ptr addrspace(10) [[TMP0]]) +; CHECK-NEXT: unreachable +; CHECK: err86: +; CHECK-NEXT: unreachable +; +top: + br i1 undef, label %err86, label %pass9 + +pass9: ; preds = %top + %0 = load atomic {} addrspace(10)*, {} addrspace(10)* addrspace(11)* undef acquire, align 8 + unreachable + +err86: ; preds = %top + unreachable +} From 5482a78570564bb3c558b91a90ad4bb783f5fa5d Mon Sep 17 00:00:00 2001 From: Shubham Sandeep Rastogi Date: Fri, 11 Mar 2022 12:14:05 -0800 Subject: [PATCH 12/34] Rename EHFrameSplitter to DWARFRecordSectionSplitter EHFrameSplitter does the exact same work to split up the eh_frame as it would need for any section that follows the DWARF record, therefore this patch just changes the name of it to DWARFRecordSectionSplitter to be more general. Differential Revision: https://reviews.llvm.org/D121486 --- .../JITLink/EHFrameSupport.cpp | 31 ++++++++++--------- .../JITLink/EHFrameSupportImpl.h | 14 ++++----- .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 2 +- .../ExecutionEngine/JITLink/MachO_arm64.cpp | 3 +- .../ExecutionEngine/JITLink/MachO_x86_64.cpp | 2 +- .../JITLink/AArch64/MachO_arm64_ehframe.s | 2 +- 6 files changed, 28 insertions(+), 26 deletions(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 2ae193595fc021..af7d959380dcc4 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -18,40 +18,41 @@ namespace llvm { namespace jitlink { -EHFrameSplitter::EHFrameSplitter(StringRef EHFrameSectionName) - : EHFrameSectionName(EHFrameSectionName) {} +DWARFRecordSectionSplitter::DWARFRecordSectionSplitter(StringRef SectionName) + : SectionName(SectionName) {} -Error EHFrameSplitter::operator()(LinkGraph &G) { - auto *EHFrame = G.findSectionByName(EHFrameSectionName); +Error DWARFRecordSectionSplitter::operator()(LinkGraph &G) { + auto *Section = G.findSectionByName(SectionName); - if (!EHFrame) { + if (!Section) { LLVM_DEBUG({ - dbgs() << "EHFrameSplitter: No " << EHFrameSectionName + dbgs() << "DWARFRecordSectionSplitter: No " << SectionName << " section. Nothing to do\n"; }); return Error::success(); } LLVM_DEBUG({ - dbgs() << "EHFrameSplitter: Processing " << EHFrameSectionName << "...\n"; + dbgs() << "DWARFRecordSectionSplitter: Processing " << SectionName + << "...\n"; }); DenseMap Caches; { // Pre-build the split caches. - for (auto *B : EHFrame->blocks()) + for (auto *B : Section->blocks()) Caches[B] = LinkGraph::SplitBlockCache::value_type(); - for (auto *Sym : EHFrame->symbols()) + for (auto *Sym : Section->symbols()) Caches[&Sym->getBlock()]->push_back(Sym); - for (auto *B : EHFrame->blocks()) + for (auto *B : Section->blocks()) llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { return LHS->getOffset() > RHS->getOffset(); }); } // Iterate over blocks (we do this by iterating over Caches entries rather - // than EHFrame->blocks() as we will be inserting new blocks along the way, + // than Section->blocks() as we will be inserting new blocks along the way, // which would invalidate iterators in the latter sequence. for (auto &KV : Caches) { auto &B = *KV.first; @@ -63,14 +64,14 @@ Error EHFrameSplitter::operator()(LinkGraph &G) { return Error::success(); } -Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, - LinkGraph::SplitBlockCache &Cache) { +Error DWARFRecordSectionSplitter::processBlock( + LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache) { LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); - // eh-frame should not contain zero-fill blocks. + // Section should not contain zero-fill blocks. if (B.isZeroFill()) return make_error("Unexpected zero-fill block in " + - EHFrameSectionName + " section"); + SectionName + " section"); if (B.getSize() == 0) { LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index ef4b47b9aa28ca..425a8afef1c988 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,19 +21,19 @@ namespace llvm { namespace jitlink { -/// A LinkGraph pass that splits blocks in an eh-frame section into sub-blocks -/// representing individual eh-frames. -/// EHFrameSplitter should not be run without EHFrameEdgeFixer, which is -/// responsible for adding FDE-to-CIE edges. -class EHFrameSplitter { +/// A LinkGraph pass that splits blocks in a section that follows the DWARF +/// Record format into sub-blocks where each header gets its own block. +/// When splitting EHFrames, DWARFRecordSectionSplitter should not be run +/// without EHFrameEdgeFixer, which is responsible for adding FDE-to-CIE edges. +class DWARFRecordSectionSplitter { public: - EHFrameSplitter(StringRef EHFrameSectionName); + DWARFRecordSectionSplitter(StringRef SectionName); Error operator()(LinkGraph &G); private: Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); - StringRef EHFrameSectionName; + StringRef SectionName; }; /// A LinkGraph pass that adds missing FDE-to-CIE, FDE-to-PC and FDE-to-LSDA diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 79d2cdbb30f18c..398a38f6c6e2bd 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -379,7 +379,7 @@ void link_ELF_x86_64(std::unique_ptr G, if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { - Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); Config.PrePrunePasses.push_back( EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64, x86_64::Delta32, x86_64::NegDelta32)); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 3ca2e40c7263f8..b4a41e68991b97 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -712,7 +712,8 @@ void link_MachO_arm64(std::unique_ptr G, // Add eh-frame passses. // FIXME: Prune eh-frames for which compact-unwind is available once // we support compact-unwind registration with libunwind. - Config.PrePrunePasses.push_back(EHFrameSplitter("__TEXT,__eh_frame")); + Config.PrePrunePasses.push_back( + DWARFRecordSectionSplitter("__TEXT,__eh_frame")); Config.PrePrunePasses.push_back( EHFrameEdgeFixer("__TEXT,__eh_frame", 8, Delta64, Delta32, NegDelta32)); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 82afaa3aa3c554..5e5bafe921e397 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -504,7 +504,7 @@ void link_MachO_x86_64(std::unique_ptr G, } LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64() { - return EHFrameSplitter("__TEXT,__eh_frame"); + return DWARFRecordSectionSplitter("__TEXT,__eh_frame"); } LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64() { diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s index 34b4830d029ae3..425f981c48471f 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s @@ -5,7 +5,7 @@ # # Check that splitting of eh-frame sections works. # -# CHECK: EHFrameSplitter: Processing __TEXT,__eh_frame... +# CHECK: DWARFRecordSectionSplitter: Processing __TEXT,__eh_frame... # CHECK: Processing block at # CHECK: Processing CFI record at # CHECK: Extracted {{.*}} section = __TEXT,__eh_frame From cb3545935d065bc5f3fa42d101d6570899be46da Mon Sep 17 00:00:00 2001 From: Shubham Sandeep Rastogi Date: Tue, 15 Mar 2022 11:36:06 -0700 Subject: [PATCH 13/34] Move DWARFRecordSectionSplitter code to its own file With 229d576b31f4071ab68c85ac4fabb78cfa502b04 the class EHFrameSplitter was renamed to DWARFRecordSectionSplitter. This change merely moves it to it's own .cpp/.h file Differential Revision: https://reviews.llvm.org/D121721 --- .../JITLink/DWARFRecordSectionSplitter.h | 35 ++++++ .../ExecutionEngine/JITLink/CMakeLists.txt | 1 + .../JITLink/DWARFRecordSectionSplitter.cpp | 117 ++++++++++++++++++ .../JITLink/EHFrameSupport.cpp | 100 +-------------- .../JITLink/EHFrameSupportImpl.h | 15 --- .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 1 + .../ExecutionEngine/JITLink/MachO_arm64.cpp | 1 + .../ExecutionEngine/JITLink/MachO_x86_64.cpp | 1 + 8 files changed, 157 insertions(+), 114 deletions(-) create mode 100644 llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h create mode 100644 llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h b/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h new file mode 100644 index 00000000000000..d748d4b0fa5927 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h @@ -0,0 +1,35 @@ +//===--------- DWARFRecordSectionSplitter.h - JITLink -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H +#define LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// A LinkGraph pass that splits blocks in a section that follows the DWARF +/// Record format into sub-blocks where each header gets its own block. +/// When splitting EHFrames, DWARFRecordSectionSplitter should not be run +/// without EHFrameEdgeFixer, which is responsible for adding FDE-to-CIE edges. +class DWARFRecordSectionSplitter { +public: + DWARFRecordSectionSplitter(StringRef SectionName); + Error operator()(LinkGraph &G); + +private: + Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); + + StringRef SectionName; +}; + +} // namespace jitlink +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_DWARFRECORDSECTIONSPLITTER_H diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt index f87b3b895137e2..5a5bdae3a1bd8a 100644 --- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_component_library(LLVMJITLink + DWARFRecordSectionSplitter.cpp EHFrameSupport.cpp JITLink.cpp JITLinkGeneric.cpp diff --git a/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp b/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp new file mode 100644 index 00000000000000..0fc366bf505f16 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp @@ -0,0 +1,117 @@ +//===-------- JITLink_DWARFRecordSectionSplitter.cpp - JITLink-------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" +#include "llvm/Support/BinaryStreamReader.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +DWARFRecordSectionSplitter::DWARFRecordSectionSplitter(StringRef SectionName) + : SectionName(SectionName) {} + +Error DWARFRecordSectionSplitter::operator()(LinkGraph &G) { + auto *Section = G.findSectionByName(SectionName); + + if (!Section) { + LLVM_DEBUG({ + dbgs() << "DWARFRecordSectionSplitter: No " << SectionName + << " section. Nothing to do\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "DWARFRecordSectionSplitter: Processing " << SectionName + << "...\n"; + }); + + DenseMap Caches; + + { + // Pre-build the split caches. + for (auto *B : Section->blocks()) + Caches[B] = LinkGraph::SplitBlockCache::value_type(); + for (auto *Sym : Section->symbols()) + Caches[&Sym->getBlock()]->push_back(Sym); + for (auto *B : Section->blocks()) + llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + + // Iterate over blocks (we do this by iterating over Caches entries rather + // than Section->blocks() as we will be inserting new blocks along the way, + // which would invalidate iterators in the latter sequence. + for (auto &KV : Caches) { + auto &B = *KV.first; + auto &BCache = KV.second; + if (auto Err = processBlock(G, B, BCache)) + return Err; + } + + return Error::success(); +} + +Error DWARFRecordSectionSplitter::processBlock( + LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache) { + LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); + + // Section should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error("Unexpected zero-fill block in " + + SectionName + " section"); + + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + BinaryStreamReader BlockReader( + StringRef(B.getContent().data(), B.getContent().size()), + G.getEndianness()); + + while (true) { + uint64_t RecordStartOffset = BlockReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress()) << "\n"; + }); + + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + if (Length != 0xffffffff) { + if (auto Err = BlockReader.skip(Length)) + return Err; + } else { + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + if (auto Err = BlockReader.skip(ExtendedLength)) + return Err; + } + + // If this was the last block then there's nothing to split + if (BlockReader.empty()) { + LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); + return Error::success(); + } + + uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; + auto &NewBlock = G.splitBlock(B, BlockSize); + (void)NewBlock; + LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); + } +} + +} // namespace jitlink +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index af7d959380dcc4..875cd1b08f8721 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -10,6 +10,7 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/Support/DynamicLibrary.h" @@ -18,105 +19,6 @@ namespace llvm { namespace jitlink { -DWARFRecordSectionSplitter::DWARFRecordSectionSplitter(StringRef SectionName) - : SectionName(SectionName) {} - -Error DWARFRecordSectionSplitter::operator()(LinkGraph &G) { - auto *Section = G.findSectionByName(SectionName); - - if (!Section) { - LLVM_DEBUG({ - dbgs() << "DWARFRecordSectionSplitter: No " << SectionName - << " section. Nothing to do\n"; - }); - return Error::success(); - } - - LLVM_DEBUG({ - dbgs() << "DWARFRecordSectionSplitter: Processing " << SectionName - << "...\n"; - }); - - DenseMap Caches; - - { - // Pre-build the split caches. - for (auto *B : Section->blocks()) - Caches[B] = LinkGraph::SplitBlockCache::value_type(); - for (auto *Sym : Section->symbols()) - Caches[&Sym->getBlock()]->push_back(Sym); - for (auto *B : Section->blocks()) - llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { - return LHS->getOffset() > RHS->getOffset(); - }); - } - - // Iterate over blocks (we do this by iterating over Caches entries rather - // than Section->blocks() as we will be inserting new blocks along the way, - // which would invalidate iterators in the latter sequence. - for (auto &KV : Caches) { - auto &B = *KV.first; - auto &BCache = KV.second; - if (auto Err = processBlock(G, B, BCache)) - return Err; - } - - return Error::success(); -} - -Error DWARFRecordSectionSplitter::processBlock( - LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache) { - LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); - - // Section should not contain zero-fill blocks. - if (B.isZeroFill()) - return make_error("Unexpected zero-fill block in " + - SectionName + " section"); - - if (B.getSize() == 0) { - LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); - return Error::success(); - } - - BinaryStreamReader BlockReader( - StringRef(B.getContent().data(), B.getContent().size()), - G.getEndianness()); - - while (true) { - uint64_t RecordStartOffset = BlockReader.getOffset(); - - LLVM_DEBUG({ - dbgs() << " Processing CFI record at " - << formatv("{0:x16}", B.getAddress()) << "\n"; - }); - - uint32_t Length; - if (auto Err = BlockReader.readInteger(Length)) - return Err; - if (Length != 0xffffffff) { - if (auto Err = BlockReader.skip(Length)) - return Err; - } else { - uint64_t ExtendedLength; - if (auto Err = BlockReader.readInteger(ExtendedLength)) - return Err; - if (auto Err = BlockReader.skip(ExtendedLength)) - return Err; - } - - // If this was the last block then there's nothing to split - if (BlockReader.empty()) { - LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); - return Error::success(); - } - - uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; - auto &NewBlock = G.splitBlock(B, BlockSize); - (void)NewBlock; - LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); - } -} - EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, unsigned PointerSize, Edge::Kind Delta64, Edge::Kind Delta32, Edge::Kind NegDelta32) diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 425a8afef1c988..54868f0f0ca129 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,21 +21,6 @@ namespace llvm { namespace jitlink { -/// A LinkGraph pass that splits blocks in a section that follows the DWARF -/// Record format into sub-blocks where each header gets its own block. -/// When splitting EHFrames, DWARFRecordSectionSplitter should not be run -/// without EHFrameEdgeFixer, which is responsible for adding FDE-to-CIE edges. -class DWARFRecordSectionSplitter { -public: - DWARFRecordSectionSplitter(StringRef SectionName); - Error operator()(LinkGraph &G); - -private: - Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); - - StringRef SectionName; -}; - /// A LinkGraph pass that adds missing FDE-to-CIE, FDE-to-PC and FDE-to-LSDA /// edges. class EHFrameEdgeFixer { diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 398a38f6c6e2bd..7648ad6e9730ca 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/JITLink/TableManager.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index b4a41e68991b97..d3db3d1e5d151d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "MachOLinkGraphBuilder.h" #include "PerGraphGOTAndPLTStubsBuilder.h" diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 5e5bafe921e397..21b815856598f3 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "MachOLinkGraphBuilder.h" From 339707005857ba8239b39371cf2e1cde681844d6 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Tue, 5 Apr 2022 11:45:25 -0700 Subject: [PATCH 14/34] [JITLink] Fix sorting bug for PC-begin candidate symbols during EH-frame fixup. The sort should have been lexicographic, but wasn't. This resulted in us choosing a common symbol at address zero over the intended target function, leading to a crash. This patch also moves sorting up to the start of the pass, which means that we only need to hold on to the canonical symbol at each address rather than a list of candidates. --- .../JITLink/EHFrameSupport.cpp | 37 +++++++++---------- .../JITLink/EHFrameSupportImpl.h | 2 +- ...achO_ehframe_canonical_symbol_comparison.s | 28 ++++++++++++++ 3 files changed, 47 insertions(+), 20 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 875cd1b08f8721..903bbdce884d19 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -50,7 +50,16 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) { // Build a map of all blocks and symbols in the text sections. We will use // these for finding / building edge targets when processing FDEs. for (auto &Sec : G.sections()) { - PC.AddrToSyms.addSymbols(Sec.symbols()); + // Just record the most-canonical symbol (for eh-frame purposes) at each + // address. + for (auto *Sym : Sec.symbols()) { + auto &CurSym = PC.AddrToSym[Sym->getAddress()]; + if (!CurSym || (std::make_tuple(Sym->getLinkage(), Sym->getScope(), + !Sym->hasName(), Sym->getName()) < + std::make_tuple(CurSym->getLinkage(), CurSym->getScope(), + !CurSym->hasName(), CurSym->getName()))) + CurSym = Sym; + } if (auto Err = PC.AddrToBlock.addBlocks(Sec.blocks(), BlockAddressMap::includeNonNull)) return Err; @@ -609,23 +618,10 @@ EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, Expected EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, orc::ExecutorAddr Addr) { - Symbol *CanonicalSym = nullptr; - - auto UpdateCanonicalSym = [&](Symbol *Sym) { - if (!CanonicalSym || Sym->getLinkage() < CanonicalSym->getLinkage() || - Sym->getScope() < CanonicalSym->getScope() || - (Sym->hasName() && !CanonicalSym->hasName()) || - Sym->getName() < CanonicalSym->getName()) - CanonicalSym = Sym; - }; - - if (auto *SymbolsAtAddr = PC.AddrToSyms.getSymbolsAt(Addr)) - for (auto *Sym : *SymbolsAtAddr) - UpdateCanonicalSym(Sym); - - // If we found an existing symbol at the given address then use it. - if (CanonicalSym) - return *CanonicalSym; + // See whether we have a canonical symbol for the given address already. + auto CanonicalSymI = PC.AddrToSym.find(Addr); + if (CanonicalSymI != PC.AddrToSym.end()) + return *CanonicalSymI->second; // Otherwise search for a block covering the address and create a new symbol. auto *B = PC.AddrToBlock.getBlockCovering(Addr); @@ -633,7 +629,10 @@ Expected EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, return make_error("No symbol or block covering address " + formatv("{0:x16}", Addr)); - return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); + auto &S = + PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); + PC.AddrToSym[S.getAddress()] = &S; + return S; } char EHFrameNullTerminator::NullTerminatorBlockContent[4] = {0, 0, 0, 0}; diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 54868f0f0ca129..66e1e8eccf0c3d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -72,7 +72,7 @@ class EHFrameEdgeFixer { LinkGraph &G; CIEInfosMap CIEInfos; BlockAddressMap AddrToBlock; - SymbolAddressMap AddrToSyms; + DenseMap AddrToSym; }; Error processBlock(ParseContext &PC, Block &B); diff --git a/llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s b/llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s new file mode 100644 index 00000000000000..da6cb77f970426 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/X86/MachO_ehframe_canonical_symbol_comparison.s @@ -0,0 +1,28 @@ +# REQUIRES: asserts +# RUN: llvm-mc -triple=x86_64-apple-macos10.9 -filetype=obj -o %t %s +# RUN: llvm-jitlink -noexec %t +# +# Verify that PC-begin candidate symbols have been sorted correctly when adding +# PC-begin edges for FDEs. In this test both _main and _X are at address zero, +# however we expect to select _main over _X as _X is common. If the sorting +# fails we'll trigger an assert in EHFrameEdgeFixer, otherwise this test will +# succeed. + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 12, 0 sdk_version 13, 0 + .globl _main + .p2align 4, 0x90 +_main: + .cfi_startproc + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + xorl %eax, %eax + popq %rbp + retq + .cfi_endproc + + .comm _X,4,2 +.subsections_via_symbols From e9468df28c8a0988432f0a316e9b78b0ff91421e Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Fri, 15 Apr 2022 11:22:26 -0700 Subject: [PATCH 15/34] [JITLink] Refactor and expand DWARF pointer encoding support. Adds support for pointer encodings commonly used in large/static models, including non-pcrel, sdata/udata8, indirect, and omit. Also refactors pointer-encoding handling to consolidate error generation inside common functions, rather than callees of those functions. --- .../JITLink/EHFrameSupport.cpp | 418 ++++++++---------- .../JITLink/EHFrameSupportImpl.h | 36 +- .../ExecutionEngine/JITLink/ELF_x86_64.cpp | 6 +- .../ExecutionEngine/JITLink/MachO_arm64.cpp | 3 +- .../ExecutionEngine/JITLink/MachO_x86_64.cpp | 3 +- .../JITLink/AArch64/MachO_arm64_ehframe.s | 4 +- ...frame_large_static_personality_encodings.s | 203 +++++++++ 7 files changed, 431 insertions(+), 242 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 903bbdce884d19..55a600d78ba85e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -20,10 +20,12 @@ namespace llvm { namespace jitlink { EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, - unsigned PointerSize, Edge::Kind Delta64, - Edge::Kind Delta32, Edge::Kind NegDelta32) + unsigned PointerSize, Edge::Kind Pointer32, + Edge::Kind Pointer64, Edge::Kind Delta32, + Edge::Kind Delta64, Edge::Kind NegDelta32) : EHFrameSectionName(EHFrameSectionName), PointerSize(PointerSize), - Delta64(Delta64), Delta32(Delta32), NegDelta32(NegDelta32) {} + Pointer32(Pointer32), Pointer64(Pointer64), Delta32(Delta32), + Delta64(Delta64), NegDelta32(NegDelta32) {} Error EHFrameEdgeFixer::operator()(LinkGraph &G) { auto *EHFrame = G.findSectionByName(EHFrameSectionName); @@ -84,10 +86,7 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) { Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { - LLVM_DEBUG({ - dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) - << "\n"; - }); + LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); // eh-frame should not contain zero-fill blocks. if (B.isZeroFill()) @@ -121,7 +120,7 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { LLVM_DEBUG({ dbgs() << " Processing CFI record at " - << formatv("{0:x16}", B.getAddress() + RecordStartOffset) << "\n"; + << (B.getAddress() + RecordStartOffset) << "\n"; }); // Get the record length. @@ -156,7 +155,7 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { if (CIEDelta == 0) { if (auto Err = processCIE(PC, B, RecordStartOffset, CIEDeltaFieldOffset + RecordRemaining, - CIEDeltaFieldOffset)) + CIEDeltaFieldOffset, BlockEdges)) return Err; } else { if (auto Err = processFDE(PC, B, RecordStartOffset, @@ -175,7 +174,8 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, - size_t CIEDeltaFieldOffset) { + size_t CIEDeltaFieldOffset, + const BlockEdgeMap &BlockEdges) { LLVM_DEBUG(dbgs() << " Record is CIE\n"); @@ -234,66 +234,59 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, if (auto Err = RecordReader.skip(1)) return Err; - uint64_t AugmentationDataLength = 0; - if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) - return Err; + if (AugInfo->AugmentationDataPresent) { - uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); + CIEInfo.AugmentationDataPresent = true; - uint8_t *NextField = &AugInfo->Fields[0]; - while (uint8_t Field = *NextField++) { - switch (Field) { - case 'L': { - CIEInfo.FDEsHaveLSDAField = true; - uint8_t LSDAPointerEncoding; - if (auto Err = RecordReader.readInteger(LSDAPointerEncoding)) - return Err; - if (!isSupportedPointerEncoding(LSDAPointerEncoding)) - return make_error( - "Unsupported LSDA pointer encoding " + - formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CIESymbol.getAddress())); - CIEInfo.LSDAPointerEncoding = LSDAPointerEncoding; - break; - } - case 'P': { - uint8_t PersonalityPointerEncoding = 0; - if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding)) - return Err; - if (PersonalityPointerEncoding != - (dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | - dwarf::DW_EH_PE_sdata4)) - return make_error( - "Unspported personality pointer " - "encoding " + - formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CIESymbol.getAddress())); - uint32_t PersonalityPointerAddress; - if (auto Err = RecordReader.readInteger(PersonalityPointerAddress)) - return Err; - break; - } - case 'R': { - uint8_t FDEPointerEncoding; - if (auto Err = RecordReader.readInteger(FDEPointerEncoding)) - return Err; - if (!isSupportedPointerEncoding(FDEPointerEncoding)) - return make_error( - "Unsupported FDE pointer encoding " + - formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CIESymbol.getAddress())); - CIEInfo.FDEPointerEncoding = FDEPointerEncoding; - break; - } - default: - llvm_unreachable("Invalid augmentation string field"); + uint64_t AugmentationDataLength = 0; + if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': + CIEInfo.LSDAPresent = true; + if (auto PE = readPointerEncoding(RecordReader, B, "LSDA")) + CIEInfo.LSDAEncoding = *PE; + else + return PE.takeError(); + break; + case 'P': { + auto PersonalityPointerEncoding = + readPointerEncoding(RecordReader, B, "personality"); + if (!PersonalityPointerEncoding) + return PersonalityPointerEncoding.takeError(); + if (auto Err = + getOrCreateEncodedPointerEdge( + PC, BlockEdges, *PersonalityPointerEncoding, RecordReader, + B, RecordOffset + RecordReader.getOffset(), "personality") + .takeError()) + return Err; + break; + } + case 'R': + if (auto PE = readPointerEncoding(RecordReader, B, "address")) { + CIEInfo.AddressEncoding = *PE; + if (CIEInfo.AddressEncoding == dwarf::DW_EH_PE_omit) + return make_error( + "Invalid address encoding DW_EH_PE_omit in CIE at " + + formatv("{0:x}", (B.getAddress() + RecordOffset).getValue())); + } else + return PE.takeError(); + break; + default: + llvm_unreachable("Invalid augmentation string field"); + } } - } - if (RecordReader.getOffset() - AugmentationDataStartOffset > - AugmentationDataLength) - return make_error("Read past the end of the augmentation " - "data while parsing fields"); + if (RecordReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error("Read past the end of the augmentation " + "data while parsing fields"); + } assert(!PC.CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); @@ -306,7 +299,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset, uint32_t CIEDelta, - BlockEdgeMap &BlockEdges) { + const BlockEdgeMap &BlockEdges) { LLVM_DEBUG(dbgs() << " Record is FDE\n"); orc::ExecutorAddr RecordAddress = B.getAddress() + RecordOffset; @@ -334,8 +327,8 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, LLVM_DEBUG({ dbgs() << " Adding edge at " - << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) - << " to CIE at: " << formatv("{0:x16}", CIEAddress) << "\n"; + << (RecordAddress + CIEDeltaFieldOffset) + << " to CIE at: " << CIEAddress << "\n"; }); if (auto CIEInfoOrErr = PC.findCIEInfo(CIEAddress)) CIEInfo = *CIEInfoOrErr; @@ -347,8 +340,8 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, } else { LLVM_DEBUG({ dbgs() << " Already has edge at " - << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) - << " to CIE at " << formatv("{0:x16}", CIEAddress) << "\n"; + << (RecordAddress + CIEDeltaFieldOffset) << " to CIE at " + << CIEAddress << "\n"; }); auto &EI = CIEEdgeItr->second; if (EI.Addend) @@ -363,107 +356,41 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, } } - { - // Process the PC-Begin field. - Block *PCBeginBlock = nullptr; - orc::ExecutorAddrDiff PCBeginFieldOffset = RecordReader.getOffset(); - auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); - if (PCEdgeItr == BlockEdges.end()) { - auto PCBeginPtrInfo = - readEncodedPointer(CIEInfo->FDEPointerEncoding, - RecordAddress + PCBeginFieldOffset, RecordReader); - if (!PCBeginPtrInfo) - return PCBeginPtrInfo.takeError(); - orc::ExecutorAddr PCBegin = PCBeginPtrInfo->first; - Edge::Kind PCBeginEdgeKind = PCBeginPtrInfo->second; - LLVM_DEBUG({ - dbgs() << " Adding edge at " - << (RecordAddress + PCBeginFieldOffset) << " to PC at " - << formatv("{0:x16}", PCBegin) << "\n"; - }); - auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); - if (!PCBeginSym) - return PCBeginSym.takeError(); - B.addEdge(PCBeginEdgeKind, RecordOffset + PCBeginFieldOffset, *PCBeginSym, - 0); - PCBeginBlock = &PCBeginSym->getBlock(); - } else { - auto &EI = PCEdgeItr->second; - LLVM_DEBUG({ - dbgs() << " Already has edge at " - << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) - << " to PC at " << formatv("{0:x16}", EI.Target->getAddress()); - if (EI.Addend) - dbgs() << " + " << formatv("{0:x16}", EI.Addend); - dbgs() << "\n"; - }); - - // Make sure the existing edge points at a defined block. - if (!EI.Target->isDefined()) { - auto EdgeAddr = RecordAddress + PCBeginFieldOffset; - return make_error("FDE edge at " + - formatv("{0:x16}", EdgeAddr) + - " points at external block"); - } - PCBeginBlock = &EI.Target->getBlock(); - if (auto Err = RecordReader.skip( - getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) - return Err; - } - + // Process the PC-Begin field. + LLVM_DEBUG({ + dbgs() << " Processing PC-begin at " + << (RecordAddress + RecordReader.getOffset()) << "\n"; + }); + if (auto PCBegin = getOrCreateEncodedPointerEdge( + PC, BlockEdges, CIEInfo->AddressEncoding, RecordReader, B, + RecordReader.getOffset(), "PC begin")) { + assert(*PCBegin && "PC-begin symbol not set"); // Add a keep-alive edge from the FDE target to the FDE to ensure that the // FDE is kept alive if its target is. - assert(PCBeginBlock && "PC-begin block not recorded"); LLVM_DEBUG({ dbgs() << " Adding keep-alive edge from target at " - << formatv("{0:x16}", PCBeginBlock->getAddress()) << " to FDE at " - << formatv("{0:x16}", RecordAddress) << "\n"; + << (*PCBegin)->getBlock().getAddress() << " to FDE at " + << RecordAddress << "\n"; }); - PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0); - } + (*PCBegin)->getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + } else + return PCBegin.takeError(); // Skip over the PC range size field. - if (auto Err = RecordReader.skip( - getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) + if (auto Err = skipEncodedPointer(CIEInfo->AddressEncoding, RecordReader)) return Err; - if (CIEInfo->FDEsHaveLSDAField) { + if (CIEInfo->AugmentationDataPresent) { uint64_t AugmentationDataSize; if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) return Err; - orc::ExecutorAddrDiff LSDAFieldOffset = RecordReader.getOffset(); - auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); - if (LSDAEdgeItr == BlockEdges.end()) { - auto LSDAPointerInfo = - readEncodedPointer(CIEInfo->LSDAPointerEncoding, - RecordAddress + LSDAFieldOffset, RecordReader); - if (!LSDAPointerInfo) - return LSDAPointerInfo.takeError(); - orc::ExecutorAddr LSDA = LSDAPointerInfo->first; - Edge::Kind LSDAEdgeKind = LSDAPointerInfo->second; - auto LSDASym = getOrCreateSymbol(PC, LSDA); - if (!LSDASym) - return LSDASym.takeError(); - LLVM_DEBUG({ - dbgs() << " Adding edge at " - << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) - << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n"; - }); - B.addEdge(LSDAEdgeKind, RecordOffset + LSDAFieldOffset, *LSDASym, 0); - } else { - LLVM_DEBUG({ - auto &EI = LSDAEdgeItr->second; - dbgs() << " Already has edge at " - << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) - << " to LSDA at " << formatv("{0:x16}", EI.Target->getAddress()); - if (EI.Addend) - dbgs() << " + " << formatv("{0:x16}", EI.Addend); - dbgs() << "\n"; - }); - if (auto Err = RecordReader.skip(AugmentationDataSize)) + if (CIEInfo->LSDAPresent) + if (auto Err = getOrCreateEncodedPointerEdge( + PC, BlockEdges, CIEInfo->LSDAEncoding, RecordReader, B, + RecordReader.getOffset(), "LSDA") + .takeError()) return Err; - } } else { LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); } @@ -512,108 +439,155 @@ EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) { return std::move(AugInfo); } -bool EHFrameEdgeFixer::isSupportedPointerEncoding(uint8_t PointerEncoding) { +Expected EHFrameEdgeFixer::readPointerEncoding(BinaryStreamReader &R, + Block &InBlock, + const char *FieldName) { using namespace dwarf; - // We only support PC-rel for now. - if ((PointerEncoding & 0x70) != DW_EH_PE_pcrel) - return false; - - // readEncodedPointer does not handle indirect. - if (PointerEncoding & DW_EH_PE_indirect) - return false; + uint8_t PointerEncoding; + if (auto Err = R.readInteger(PointerEncoding)) + return Err; - // Supported datatypes. + bool Supported = true; switch (PointerEncoding & 0xf) { - case DW_EH_PE_absptr: - case DW_EH_PE_udata4: - case DW_EH_PE_udata8: - case DW_EH_PE_sdata4: - case DW_EH_PE_sdata8: - return true; + case DW_EH_PE_uleb128: + case DW_EH_PE_udata2: + case DW_EH_PE_sleb128: + case DW_EH_PE_sdata2: + Supported = false; + break; + } + if (Supported) { + switch (PointerEncoding & 0x70) { + case DW_EH_PE_textrel: + case DW_EH_PE_datarel: + case DW_EH_PE_funcrel: + case DW_EH_PE_aligned: + Supported = false; + break; + } } - return false; + if (Supported) + return PointerEncoding; + + return make_error("Unsupported pointer encoding " + + formatv("{0:x2}", PointerEncoding) + " for " + + FieldName + "in CFI record at " + + formatv("{0:x16}", InBlock.getAddress())); } -unsigned EHFrameEdgeFixer::getPointerEncodingDataSize(uint8_t PointerEncoding) { +Error EHFrameEdgeFixer::skipEncodedPointer(uint8_t PointerEncoding, + BinaryStreamReader &RecordReader) { using namespace dwarf; - assert(isSupportedPointerEncoding(PointerEncoding) && - "Unsupported pointer encoding"); + // Switch absptr to corresponding udata encoding. + if ((PointerEncoding & 0xf) == DW_EH_PE_absptr) + PointerEncoding |= (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + switch (PointerEncoding & 0xf) { - case DW_EH_PE_absptr: - return PointerSize; case DW_EH_PE_udata4: case DW_EH_PE_sdata4: - return 4; + if (auto Err = RecordReader.skip(4)) + return Err; + break; case DW_EH_PE_udata8: case DW_EH_PE_sdata8: - return 8; + if (auto Err = RecordReader.skip(8)) + return Err; + break; default: - llvm_unreachable("Unsupported encoding"); + llvm_unreachable("Unrecognized encoding"); } + return Error::success(); } -Expected> -EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, - orc::ExecutorAddr PointerFieldAddress, - BinaryStreamReader &RecordReader) { - assert(isSupportedPointerEncoding(PointerEncoding) && - "Unsupported pointer encoding"); - +Expected EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( + ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding, + BinaryStreamReader &RecordReader, Block &BlockToFix, + size_t PointerFieldOffset, const char *FieldName) { using namespace dwarf; - // Isolate data type, remap absptr to udata4 or udata8. This relies on us - // having verified that the graph uses 32-bit or 64-bit pointers only at the - // start of this pass. - uint8_t EffectiveType = PointerEncoding & 0xf; - if (EffectiveType == DW_EH_PE_absptr) - EffectiveType = (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + if (PointerEncoding == DW_EH_PE_omit) + return nullptr; - orc::ExecutorAddr Addr; - Edge::Kind PointerEdgeKind = Edge::Invalid; - switch (EffectiveType) { + // If there's already an edge here then just skip the encoded pointer and + // return the edge's target. + { + auto EdgeI = BlockEdges.find(PointerFieldOffset); + if (EdgeI != BlockEdges.end()) { + LLVM_DEBUG({ + dbgs() << " Existing edge at " + << (BlockToFix.getAddress() + PointerFieldOffset) << " to " + << FieldName << " at " << EdgeI->second.Target->getAddress(); + if (EdgeI->second.Target->hasName()) + dbgs() << " (" << EdgeI->second.Target->getName() << ")"; + dbgs() << "\n"; + }); + if (auto Err = skipEncodedPointer(PointerEncoding, RecordReader)) + return Err; + return EdgeI->second.Target; + } + } + + // Switch absptr to corresponding udata encoding. + if ((PointerEncoding & 0xf) == DW_EH_PE_absptr) + PointerEncoding |= (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + + // We need to create an edge. Start by reading the field value. + uint64_t FieldValue; + bool Is64Bit = false; + switch (PointerEncoding & 0xf) { case DW_EH_PE_udata4: { uint32_t Val; if (auto Err = RecordReader.readInteger(Val)) return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta32; - break; - } - case DW_EH_PE_udata8: { - uint64_t Val; - if (auto Err = RecordReader.readInteger(Val)) - return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta64; + FieldValue = Val; break; } case DW_EH_PE_sdata4: { - int32_t Val; + uint32_t Val; if (auto Err = RecordReader.readInteger(Val)) return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta32; + FieldValue = Val; break; } - case DW_EH_PE_sdata8: { - int64_t Val; - if (auto Err = RecordReader.readInteger(Val)) + case DW_EH_PE_udata8: + case DW_EH_PE_sdata8: + Is64Bit = true; + if (auto Err = RecordReader.readInteger(FieldValue)) return std::move(Err); - Addr = PointerFieldAddress + Val; - PointerEdgeKind = Delta64; break; - } + default: + llvm_unreachable("Unsupported encoding"); } - if (PointerEdgeKind == Edge::Invalid) - return make_error( - "Unspported edge kind for encoded pointer at " + - formatv("{0:x}", PointerFieldAddress)); + // Find the edge target and edge kind to use. + orc::ExecutorAddr Target; + Edge::Kind PtrEdgeKind = Edge::Invalid; + if ((PointerEncoding & 0x70) == DW_EH_PE_pcrel) { + Target = BlockToFix.getAddress() + PointerFieldOffset; + PtrEdgeKind = Is64Bit ? Delta64 : Delta32; + } else + PtrEdgeKind = Is64Bit ? Pointer64 : Pointer32; + Target += FieldValue; + + // Find or create a symbol to point the edge at. + auto TargetSym = getOrCreateSymbol(PC, Target); + if (!TargetSym) + return TargetSym.takeError(); + BlockToFix.addEdge(PtrEdgeKind, PointerFieldOffset, *TargetSym, 0); + + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << (BlockToFix.getAddress() + PointerFieldOffset) << " to " + << FieldName << " at " << TargetSym->getAddress(); + if (TargetSym->hasName()) + dbgs() << " (" << TargetSym->getName() << ")"; + dbgs() << "\n"; + }); - return std::make_pair(Addr, Delta64); + return &*TargetSym; } Expected EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 66e1e8eccf0c3d..55cf7fc63ee795 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -25,8 +25,12 @@ namespace jitlink { /// edges. class EHFrameEdgeFixer { public: + /// Create an eh-frame edge fixer. + /// If a given edge-kind is not supported on the target architecture then + /// Edge::Invalid should be used. EHFrameEdgeFixer(StringRef EHFrameSectionName, unsigned PointerSize, - Edge::Kind Delta64, Edge::Kind Delta32, + Edge::Kind Pointer32, Edge::Kind Pointer64, + Edge::Kind Delta32, Edge::Kind Delta64, Edge::Kind NegDelta32); Error operator()(LinkGraph &G); @@ -42,9 +46,10 @@ class EHFrameEdgeFixer { CIEInformation() = default; CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} Symbol *CIESymbol = nullptr; - bool FDEsHaveLSDAField = false; - uint8_t FDEPointerEncoding = 0; - uint8_t LSDAPointerEncoding = 0; + bool AugmentationDataPresent = false; + bool LSDAPresent = false; + uint8_t LSDAEncoding = 0; + uint8_t AddressEncoding = 0; }; struct EdgeTarget { @@ -77,28 +82,33 @@ class EHFrameEdgeFixer { Error processBlock(ParseContext &PC, Block &B); Error processCIE(ParseContext &PC, Block &B, size_t RecordOffset, - size_t RecordLength, size_t CIEDeltaFieldOffset); + size_t RecordLength, size_t CIEDeltaFieldOffset, + const BlockEdgeMap &BlockEdges); Error processFDE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset, - uint32_t CIEDelta, BlockEdgeMap &BlockEdges); + uint32_t CIEDelta, const BlockEdgeMap &BlockEdges); Expected parseAugmentationString(BinaryStreamReader &RecordReader); - static bool isSupportedPointerEncoding(uint8_t PointerEncoding); - unsigned getPointerEncodingDataSize(uint8_t PointerEncoding); - Expected> - readEncodedPointer(uint8_t PointerEncoding, - orc::ExecutorAddr PointerFieldAddress, - BinaryStreamReader &RecordReader); + Expected readPointerEncoding(BinaryStreamReader &RecordReader, + Block &InBlock, const char *FieldName); + Error skipEncodedPointer(uint8_t PointerEncoding, + BinaryStreamReader &RecordReader); + Expected getOrCreateEncodedPointerEdge( + ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding, + BinaryStreamReader &RecordReader, Block &BlockToFix, + size_t PointerFieldOffset, const char *FieldName); Expected getOrCreateSymbol(ParseContext &PC, orc::ExecutorAddr Addr); StringRef EHFrameSectionName; unsigned PointerSize; - Edge::Kind Delta64; + Edge::Kind Pointer32; + Edge::Kind Pointer64; Edge::Kind Delta32; + Edge::Kind Delta64; Edge::Kind NegDelta32; }; diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 7648ad6e9730ca..ca9a2964ff146b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -381,9 +381,9 @@ void link_ELF_x86_64(std::unique_ptr G, if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); - Config.PrePrunePasses.push_back( - EHFrameEdgeFixer(".eh_frame", x86_64::PointerSize, x86_64::Delta64, - x86_64::Delta32, x86_64::NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64, + x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32)); Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); // Construct a JITLinker and run the link function. diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index d3db3d1e5d151d..873c820e5be497 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -716,7 +716,8 @@ void link_MachO_arm64(std::unique_ptr G, Config.PrePrunePasses.push_back( DWARFRecordSectionSplitter("__TEXT,__eh_frame")); Config.PrePrunePasses.push_back( - EHFrameEdgeFixer("__TEXT,__eh_frame", 8, Delta64, Delta32, NegDelta32)); + EHFrameEdgeFixer("__TEXT,__eh_frame", 8, Pointer32, Pointer64, Delta32, + Delta64, NegDelta32)); // Add an in-place GOT/Stubs pass. Config.PostPrunePasses.push_back( diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 21b815856598f3..2f61b31ae38328 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -510,7 +510,8 @@ LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64() { LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64() { return EHFrameEdgeFixer("__TEXT,__eh_frame", x86_64::PointerSize, - x86_64::Delta64, x86_64::Delta32, x86_64::NegDelta32); + x86_64::Pointer32, x86_64::Pointer64, x86_64::Delta32, + x86_64::Delta64, x86_64::NegDelta32); } } // end namespace jitlink diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s index 425f981c48471f..6ae5973fb82307 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_arm64_ehframe.s @@ -19,9 +19,9 @@ # CHECK: Processing CFI record at # CHECK: Record is FDE # CHECK: Adding edge at {{.*}} to CIE at: {{.*}} -# CHECK: Already has edge at {{.*}} to PC at {{.*}} +# CHECK: Existing edge at {{.*}} to PC begin at {{.*}} # CHECK: Adding keep-alive edge from target at {{.*}} to FDE at {{.*}} -# CHECK: Already has edge at {{.*}} to LSDA at {{.*}} +# CHECK: Existing edge at {{.*}} to LSDA at {{.*}} .section __TEXT,__text,regular,pure_instructions .globl _main diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s new file mode 100644 index 00000000000000..83e46f8a12d427 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_ehframe_large_static_personality_encodings.s @@ -0,0 +1,203 @@ +# REQUIRES: asserts +# UNSUPPORTED: system-windows +# RUN: llvm-mc -triple=x86_64-pc-linux-gnu -large-code-model \ +# RUN: -filetype=obj -o %t %s +# RUN: llvm-jitlink -debug-only=jitlink -noexec %t 2>&1 | FileCheck %s +# +# Check handling of pointer encodings for personality functions when compiling +# with `-mcmodel=large -static`. +# + +# CHECK: Record is CIE +# CHECK-NEXT: edge at {{.*}} to personality at {{.*}} (DW.ref.__gxx_personality_v0) +# CHECK: Record is CIE +# CHECK-NEXT: edge at {{.*}} to personality at {{.*}} (__gxx_personality_v0) + + .text + .file "eh.cpp" + + .globl main + .p2align 4, 0x90 + .type main,@function +main: + xorl %eax, %eax + retq +.Lfunc_end_main: + .size main, .Lfunc_end_main-main + +# pe_absptr uses absptr encoding for __gxx_personality_v0 + .text + .globl pe_absptr + .p2align 4, 0x90 + .type pe_absptr,@function +pe_absptr: +.Lfunc_begin0: + .cfi_startproc + .cfi_personality 0, __gxx_personality_v0 + .cfi_lsda 0, .Lexception0 + pushq %rax + .cfi_def_cfa_offset 16 + movabsq $__cxa_allocate_exception, %rax + movl $4, %edi + callq *%rax + movl $42, (%rax) +.Ltmp0: + movabsq $_ZTIi, %rsi + movabsq $__cxa_throw, %rcx + movq %rax, %rdi + xorl %edx, %edx + callq *%rcx +.Ltmp1: +.LBB0_2: +.Ltmp2: + movabsq $__cxa_begin_catch, %rcx + movq %rax, %rdi + callq *%rcx + movabsq $__cxa_end_catch, %rax + popq %rcx + .cfi_def_cfa_offset 8 + jmpq *%rax +.Lfunc_end0: + .size pe_absptr, .Lfunc_end0-pe_absptr + .cfi_endproc + .section .gcc_except_table,"a",@progbits + .p2align 2 +GCC_except_table0: +.Lexception0: + .byte 255 # @LPStart Encoding = omit + .byte 0 # @TType Encoding = absptr + .uleb128 .Lttbase0-.Lttbaseref0 +.Lttbaseref0: + .byte 1 # Call site Encoding = uleb128 + .uleb128 .Lcst_end0-.Lcst_begin0 +.Lcst_begin0: + .uleb128 .Lfunc_begin0-.Lfunc_begin0 # >> Call Site 1 << + .uleb128 .Ltmp0-.Lfunc_begin0 # Call between .Lfunc_begin0 and .Ltmp0 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup + .uleb128 .Ltmp0-.Lfunc_begin0 # >> Call Site 2 << + .uleb128 .Ltmp1-.Ltmp0 # Call between .Ltmp0 and .Ltmp1 + .uleb128 .Ltmp2-.Lfunc_begin0 # jumps to .Ltmp2 + .byte 1 # On action: 1 + .uleb128 .Ltmp1-.Lfunc_begin0 # >> Call Site 3 << + .uleb128 .Lfunc_end0-.Ltmp1 # Call between .Ltmp1 and .Lfunc_end0 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup +.Lcst_end0: + .byte 1 # >> Action Record 1 << + # Catch TypeInfo 1 + .byte 0 # No further actions + .p2align 2 + # >> Catch TypeInfos << + .quad _ZTIi # TypeInfo 1 +.Lttbase0: + .p2align 2 + # -- End function + +# pe_indir_pcrel_sdata8 uses 0x9C -- Indirect, pc-rel, sdata8 encoding to +# DW.ref.__gxx_personality_v0 + .text + .globl pe_indir_pcrel_sdata8 + .p2align 4, 0x90 + .type pe_indir_pcrel_sdata8,@function +pe_indir_pcrel_sdata8: +.Lfunc_begin1: + .cfi_startproc + .cfi_personality 156, DW.ref.__gxx_personality_v0 + .cfi_lsda 28, .Lexception1 + pushq %r14 + .cfi_def_cfa_offset 16 + pushq %rbx + .cfi_def_cfa_offset 24 + pushq %rax + .cfi_def_cfa_offset 32 + .cfi_offset %rbx, -24 + .cfi_offset %r14, -16 +.L1$pb: + leaq .L1$pb(%rip), %rax + movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rbx + addq %rax, %rbx + movabsq $__cxa_allocate_exception@GOT, %rax + movl $4, %edi + callq *(%rbx,%rax) + movl $42, (%rax) +.Ltmp4: + movabsq $_ZTIi@GOT, %rcx + movq (%rbx,%rcx), %rsi + movabsq $__cxa_throw@GOT, %rcx + movq %rax, %rdi + xorl %edx, %edx + movq %rbx, %r14 + callq *(%rbx,%rcx) +.Ltmp5: +.LBB1_2: +.Ltmp6: + movabsq $__cxa_begin_catch@GOT, %rcx + movq %rax, %rdi + callq *(%r14,%rcx) + movabsq $__cxa_end_catch@GOT, %rax + movq %r14, %rcx + addq $8, %rsp + .cfi_def_cfa_offset 24 + popq %rbx + .cfi_def_cfa_offset 16 + popq %r14 + .cfi_def_cfa_offset 8 + jmpq *(%rcx,%rax) +.Lfunc_end1: + .size pe_indir_pcrel_sdata8, .Lfunc_end1-pe_indir_pcrel_sdata8 + .cfi_endproc + .section .gcc_except_table,"a",@progbits + .p2align 2 +GCC_except_table1: +.Lexception1: + .byte 255 # @LPStart Encoding = omit + .byte 156 # @TType Encoding = indirect pcrel sdata8 + .uleb128 .Lttbase1-.Lttbaseref1 +.Lttbaseref1: + .byte 1 # Call site Encoding = uleb128 + .uleb128 .Lcst_end1-.Lcst_begin1 +.Lcst_begin1: + .uleb128 .Lfunc_begin1-.Lfunc_begin1 # >> Call Site 1 << + .uleb128 .Ltmp4-.Lfunc_begin1 # Call between .Lfunc_begin1 and .Ltmp4 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup + .uleb128 .Ltmp4-.Lfunc_begin1 # >> Call Site 2 << + .uleb128 .Ltmp5-.Ltmp4 # Call between .Ltmp4 and .Ltmp5 + .uleb128 .Ltmp6-.Lfunc_begin1 # jumps to .Ltmp6 + .byte 1 # On action: 1 + .uleb128 .Ltmp5-.Lfunc_begin1 # >> Call Site 3 << + .uleb128 .Lfunc_end1-.Ltmp5 # Call between .Ltmp5 and .Lfunc_end1 + .byte 0 # has no landing pad + .byte 0 # On action: cleanup +.Lcst_end1: + .byte 1 # >> Action Record 1 << + # Catch TypeInfo 1 + .byte 0 # No further actions + .p2align 2 + # >> Catch TypeInfos << +.Ltmp7: # TypeInfo 1 + .quad .L_ZTIi.DW.stub-.Ltmp7 +.Lttbase1: + .p2align 2 + # -- End function + + .data + .p2align 3 +.L_ZTIi.DW.stub: + .quad _ZTIi + .hidden DW.ref.__gxx_personality_v0 + .weak DW.ref.__gxx_personality_v0 + + .section .data.DW.ref.__gxx_personality_v0,"aGw",@progbits,DW.ref.__gxx_personality_v0,comdat + .p2align 3 + .type DW.ref.__gxx_personality_v0,@object + .size DW.ref.__gxx_personality_v0, 8 +DW.ref.__gxx_personality_v0: + .quad __gxx_personality_v0 + + .ident "clang version 13.0.1" + .section ".note.GNU-stack","",@progbits + .addrsig + .addrsig_sym __gxx_personality_v0 + .addrsig_sym _ZTIi From 2d754c8c522f63bd0bb7cda24d69f84d377d2e3c Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Fri, 15 Apr 2022 12:57:19 -0700 Subject: [PATCH 16/34] [JITLink] Add missing moves from 43acef48d38e. --- llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 55a600d78ba85e..8f5c990e003600 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -446,7 +446,7 @@ Expected EHFrameEdgeFixer::readPointerEncoding(BinaryStreamReader &R, uint8_t PointerEncoding; if (auto Err = R.readInteger(PointerEncoding)) - return Err; + return std::move(Err); bool Supported = true; switch (PointerEncoding & 0xf) { @@ -525,7 +525,7 @@ Expected EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( dbgs() << "\n"; }); if (auto Err = skipEncodedPointer(PointerEncoding, RecordReader)) - return Err; + return std::move(Err); return EdgeI->second.Target; } } From b1d39fc04e160e01535810e41d60f494e99fa660 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sat, 16 Apr 2022 18:49:47 -0700 Subject: [PATCH 17/34] [JITLink] Error instead of asserting on unrecognized edge kinds. It's idiomatic to require that plugins (especially platform plugins) be installed to handle special edge kinds. If the plugins are not installed and an object is loaded that uses one of the special edge kinds then we want to error out rather than asserting. --- llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h | 9 ++++----- llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp | 5 ++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h index 4a4e8d15be6640..9a2bc9b09350eb 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -447,11 +447,10 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, break; } - default: { - // If you hit this you should check that *constructor and other non-fixup - // edges have been removed prior to applying fixups. - llvm_unreachable("Graph contains edge kind with no fixup expression"); - } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); } return Error::success(); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 873c820e5be497..ed59aa4be96870 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -677,7 +677,10 @@ class MachOJITLinker_arm64 : public JITLinker { break; } default: - llvm_unreachable("Unrecognized edge kind"); + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + + getMachOARM64RelocationKindName(E.getKind())); } return Error::success(); From 4b51c03db8b582408034d55cfe5837d0c951621f Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 7 Jun 2022 11:39:05 -0700 Subject: [PATCH 18/34] [JITLink][ELF][AArch64] Lift MachO/arm64 edges into aarch64.h, reuse for ELF. This patch moves the aarch64 fixup logic from the MachO/arm64 backend to aarch64.h header so that it can be re-used in the ELF/aarch64 backend. This significantly expands relocation support in the ELF/aarch64 backend. Reviewed By: lhames, sgraenitz Differential Revision: https://reviews.llvm.org/D126286 --- .../ExecutionEngine/JITLink/MachO_arm64.h | 27 -- .../llvm/ExecutionEngine/JITLink/aarch64.h | 26 +- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 67 ++- .../ExecutionEngine/JITLink/MachO_arm64.cpp | 422 +++++++----------- llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 199 ++++++++- 5 files changed, 404 insertions(+), 337 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h index aee14c0d1fe5e8..6f2ff012697ddb 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -18,30 +18,6 @@ namespace llvm { namespace jitlink { -namespace MachO_arm64_Edges { - -enum MachOARM64RelocationKind : Edge::Kind { - Branch26 = Edge::FirstRelocation, - Pointer32, - Pointer64, - Pointer64Anon, - Page21, - PageOffset12, - GOTPage21, - GOTPageOffset12, - TLVPage21, - TLVPageOffset12, - PointerToGOT, - PairedAddend, - LDRLiteral19, - Delta32, - Delta64, - NegDelta32, - NegDelta64, -}; - -} // namespace MachO_arm64_Edges - /// Create a LinkGraph from a MachO/arm64 relocatable object. /// /// Note: The graph does not take ownership of the underlying buffer, nor copy @@ -62,9 +38,6 @@ createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer); void link_MachO_arm64(std::unique_ptr G, std::unique_ptr Ctx); -/// Return the string name of the given MachO arm64 edge kind. -const char *getMachOARM64RelocationKindName(Edge::Kind R); - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 994ce783b05863..ef38a1690c9b2b 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -19,18 +19,34 @@ namespace llvm { namespace jitlink { namespace aarch64 { -/// Represets aarch64 fixups enum EdgeKind_aarch64 : Edge::Kind { - - /// Set a CALL immediate field to bits [27:2] of X = Target - Fixup + Addend - R_AARCH64_CALL26 = Edge::FirstRelocation, - + Branch26 = Edge::FirstRelocation, + Pointer32, + Pointer64, + Pointer64Anon, + Page21, + PageOffset12, + GOTPage21, + GOTPageOffset12, + TLVPage21, + TLVPageOffset12, + PointerToGOT, + PairedAddend, + LDRLiteral19, + Delta32, + Delta64, + NegDelta32, + NegDelta64, }; /// Returns a string name for the given aarch64 edge. For debugging purposes /// only const char *getEdgeKindName(Edge::Kind K); +unsigned getPageOffset12Shift(uint32_t Instr); + +Error applyFixup(LinkGraph &G, Block &B, const Edge &E); + } // namespace aarch64 } // namespace jitlink } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index dd3eb97c21a0ea..1d00dab8212c31 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -37,46 +37,23 @@ class ELFJITLinker_aarch64 : public JITLinker { private: Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { - using namespace aarch64; - using namespace llvm::support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - auto FixupAddress = B.getAddress() + E.getOffset(); - switch (E.getKind()) { - case aarch64::R_AARCH64_CALL26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Call-inst is not 32-bit aligned"); - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Call target is not 32-bit aligned"); - - if (!isInt<28>(Value)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - } - return Error::success(); + return aarch64::applyFixup(G, B, E); } }; template class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { private: - static Expected + enum ELFAArch64RelocationKind : Edge::Kind { + ELFBranch26 = Edge::FirstRelocation, + }; + + static Expected getRelocationKind(const uint32_t Type) { using namespace aarch64; switch (Type) { case ELF::R_AARCH64_CALL26: - return EdgeKind_aarch64::R_AARCH64_CALL26; + return ELFBranch26; } return make_error("Unsupported aarch64 relocation:" + @@ -116,18 +93,28 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { inconvertibleErrorCode()); uint32_t Type = Rel.getType(false); - Expected Kind = getRelocationKind(Type); - if (!Kind) - return Kind.takeError(); + Expected RelocKind = getRelocationKind(Type); + if (!RelocKind) + return RelocKind.takeError(); int64_t Addend = Rel.r_addend; orc::ExecutorAddr FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); - Edge GE(*Kind, Offset, *GraphSymbol, Addend); + + Edge::Kind Kind = Edge::Invalid; + + switch (*RelocKind) { + case ELFBranch26: { + Kind = aarch64::Branch26; + break; + } + }; + + Edge GE(Kind, Offset, *GraphSymbol, Addend); LLVM_DEBUG({ dbgs() << " "; - printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(*Kind)); + printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind)); dbgs() << "\n"; }); @@ -135,6 +122,16 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return Error::success(); } + /// Return the string name of the given ELF aarch64 edge kind. + const char *getELFAArch64RelocationKindName(Edge::Kind R) { + switch (R) { + case ELFBranch26: + return "ELFBranch26"; + default: + return getGenericEdgeKindName(static_cast(R)); + } + } + public: ELFLinkGraphBuilder_aarch64(StringRef FileName, const object::ELFFile &Obj, const Triple T) diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index ed59aa4be96870..1356f53b629220 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -12,6 +12,7 @@ #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" +#include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "MachOLinkGraphBuilder.h" #include "PerGraphGOTAndPLTStubsBuilder.h" @@ -20,7 +21,6 @@ using namespace llvm; using namespace llvm::jitlink; -using namespace llvm::jitlink::MachO_arm64_Edges; namespace { @@ -28,19 +28,39 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin"), - getMachOARM64RelocationKindName), + aarch64::getEdgeKindName), NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: + enum MachOARM64RelocationKind : Edge::Kind { + MachOBranch26 = Edge::FirstRelocation, + MachOPointer32, + MachOPointer64, + MachOPointer64Anon, + MachOPage21, + MachOPageOffset12, + MachOGOTPage21, + MachOGOTPageOffset12, + MachOTLVPage21, + MachOTLVPageOffset12, + MachOPointerToGOT, + MachOPairedAddend, + MachOLDRLiteral19, + MachODelta32, + MachODelta64, + MachONegDelta32, + MachONegDelta64, + }; + static Expected getRelocationKind(const MachO::relocation_info &RI) { switch (RI.r_type) { case MachO::ARM64_RELOC_UNSIGNED: if (!RI.r_pcrel) { if (RI.r_length == 3) - return RI.r_extern ? Pointer64 : Pointer64Anon; + return RI.r_extern ? MachOPointer64 : MachOPointer64Anon; else if (RI.r_length == 2) - return Pointer32; + return MachOPointer32; } break; case MachO::ARM64_RELOC_SUBTRACTOR: @@ -49,46 +69,46 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { // They may be turned into NegDelta by parsePairRelocation. if (!RI.r_pcrel && RI.r_extern) { if (RI.r_length == 2) - return Delta32; + return MachODelta32; else if (RI.r_length == 3) - return Delta64; + return MachODelta64; } break; case MachO::ARM64_RELOC_BRANCH26: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return Branch26; + return MachOBranch26; break; case MachO::ARM64_RELOC_PAGE21: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return Page21; + return MachOPage21; break; case MachO::ARM64_RELOC_PAGEOFF12: if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PageOffset12; + return MachOPageOffset12; break; case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return GOTPage21; + return MachOGOTPage21; break; case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return GOTPageOffset12; + return MachOGOTPageOffset12; break; case MachO::ARM64_RELOC_POINTER_TO_GOT: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PointerToGOT; + return MachOPointerToGOT; break; case MachO::ARM64_RELOC_ADDEND: if (!RI.r_pcrel && !RI.r_extern && RI.r_length == 2) - return PairedAddend; + return MachOPairedAddend; break; case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return TLVPage21; + return MachOTLVPage21; break; case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return TLVPageOffset12; + return MachOTLVPageOffset12; break; } @@ -102,8 +122,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { ", length=" + formatv("{0:d}", RI.r_length)); } - using PairRelocInfo = - std::tuple; + using PairRelocInfo = std::tuple; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. @@ -115,8 +134,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { object::relocation_iterator &RelEnd) { using namespace support; - assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || - (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + assert(((SubtractorKind == MachODelta32 && SubRI.r_length == 2) || + (SubtractorKind == MachODelta64 && SubRI.r_length == 3)) && "Subtractor kind should match length"); assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); @@ -166,17 +185,18 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { FixupValue -= ToSymbol->getAddress().getValue(); } - MachOARM64RelocationKind DeltaKind; + Edge::Kind DeltaKind; Symbol *TargetSymbol; uint64_t Addend; if (&BlockToFix == &FromSymbol->getAddressable()) { TargetSymbol = ToSymbol; - DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + DeltaKind = (SubRI.r_length == 3) ? aarch64::Delta64 : aarch64::Delta32; Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. } else if (&BlockToFix == &ToSymbol->getAddressable()) { TargetSymbol = &*FromSymbol; - DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + DeltaKind = + (SubRI.r_length == 3) ? aarch64::NegDelta64 : aarch64::NegDelta32; Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { // BlockToFix was neither FromSymbol nor ToSymbol. @@ -230,9 +250,9 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { MachO::relocation_info RI = getRelocationInfo(RelItr); // Validate the relocation kind. - auto Kind = getRelocationKind(RI); - if (!Kind) - return Kind.takeError(); + auto MachORelocKind = getRelocationKind(RI); + if (!MachORelocKind) + return MachORelocKind.takeError(); // Find the address of the value to fix up. orc::ExecutorAddr FixupAddress = @@ -256,6 +276,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { return make_error( "Relocation content extends past end of fixup block"); + Edge::Kind Kind = Edge::Invalid; + // Get a pointer to the fixup content. const char *FixupContent = BlockToFix->getContent().data() + (FixupAddress - BlockToFix->getAddress()); @@ -264,7 +286,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; - if (*Kind == PairedAddend) { + if (*MachORelocKind == MachOPairedAddend) { // If this is an Addend relocation then process it and move to the // paired reloc. @@ -276,19 +298,21 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { ++RelItr; RI = getRelocationInfo(RelItr); - Kind = getRelocationKind(RI); - if (!Kind) - return Kind.takeError(); + MachORelocKind = getRelocationKind(RI); + if (!MachORelocKind) + return MachORelocKind.takeError(); - if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12) + if (*MachORelocKind != MachOBranch26 && + *MachORelocKind != MachOPage21 && + *MachORelocKind != MachOPageOffset12) return make_error( "Invalid relocation pair: Addend + " + - StringRef(getMachOARM64RelocationKindName(*Kind))); + StringRef(getMachOARM64RelocationKindName(*MachORelocKind))); LLVM_DEBUG({ dbgs() << " Addend: value = " << formatv("{0:x6}", Addend) - << ", pair is " << getMachOARM64RelocationKindName(*Kind) - << "\n"; + << ", pair is " + << getMachOARM64RelocationKindName(*MachORelocKind) << "\n"; }); // Find the address of the value to fix up. @@ -299,8 +323,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { "different target"); } - switch (*Kind) { - case Branch26: { + switch (*MachORelocKind) { + case MachOBranch26: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -309,23 +333,26 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { if ((Instr & 0x7fffffff) != 0x14000000) return make_error("BRANCH26 target is not a B or BL " "instruction with a zero addend"); + Kind = aarch64::Branch26; break; } - case Pointer32: + case MachOPointer32: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; + Kind = aarch64::Pointer32; break; - case Pointer64: + case MachOPointer64: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; + Kind = aarch64::Pointer64; break; - case Pointer64Anon: { + case MachOPointer64Anon: { orc::ExecutorAddr TargetAddress(*(const ulittle64_t *)FixupContent); auto TargetNSec = findSectionByIndex(RI.r_symbolnum - 1); if (!TargetNSec) @@ -336,11 +363,12 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { else return TargetSymbolOrErr.takeError(); Addend = TargetAddress - TargetSymbol->getAddress(); + Kind = aarch64::Pointer64Anon; break; } - case Page21: - case TLVPage21: - case GOTPage21: { + case MachOPage21: + case MachOTLVPage21: + case MachOGOTPage21: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -350,9 +378,17 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { return make_error("PAGE21/GOTPAGE21 target is not an " "ADRP instruction with a zero " "addend"); + + if (*MachORelocKind == MachOPage21) { + Kind = aarch64::Page21; + } else if (*MachORelocKind == MachOTLVPage21) { + Kind = aarch64::TLVPage21; + } else if (*MachORelocKind == MachOGOTPage21) { + Kind = aarch64::GOTPage21; + } break; } - case PageOffset12: { + case MachOPageOffset12: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -362,10 +398,11 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { if (EncodedAddend != 0) return make_error("GOTPAGEOFF12 target has non-zero " "encoded addend"); + Kind = aarch64::PageOffset12; break; } - case TLVPageOffset12: - case GOTPageOffset12: { + case MachOTLVPageOffset12: + case MachOGOTPageOffset12: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -375,27 +412,35 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { return make_error("GOTPAGEOFF12 target is not an LDR " "immediate instruction with a zero " "addend"); + + if (*MachORelocKind == MachOTLVPageOffset12) { + Kind = aarch64::TLVPageOffset12; + } else if (*MachORelocKind == MachOGOTPageOffset12) { + Kind = aarch64::GOTPageOffset12; + } break; } - case PointerToGOT: + case MachOPointerToGOT: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); + + Kind = aarch64::PointerToGOT; break; - case Delta32: - case Delta64: { + case MachODelta32: + case MachODelta64: { // We use Delta32/Delta64 to represent SUBTRACTOR relocations. // parsePairRelocation handles the paired reloc, and returns the // edge kind to be used (either Delta32/Delta64, or // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, - FixupContent, ++RelItr, RelEnd); + parsePairRelocation(*BlockToFix, *MachORelocKind, RI, + FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + std::tie(Kind, TargetSymbol, Addend) = *PairInfo; assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } @@ -406,19 +451,58 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { LLVM_DEBUG({ dbgs() << " "; - Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *BlockToFix, GE, - getMachOARM64RelocationKindName(*Kind)); + printEdge(dbgs(), *BlockToFix, GE, aarch64::getEdgeKindName(Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); } } return Error::success(); } + /// Return the string name of the given MachO arm64 edge kind. + const char *getMachOARM64RelocationKindName(Edge::Kind R) { + switch (R) { + case MachOBranch26: + return "MachOBranch26"; + case MachOPointer64: + return "MachOPointer64"; + case MachOPointer64Anon: + return "MachOPointer64Anon"; + case MachOPage21: + return "MachOPage21"; + case MachOPageOffset12: + return "MachOPageOffset12"; + case MachOGOTPage21: + return "MachOGOTPage21"; + case MachOGOTPageOffset12: + return "MachOGOTPageOffset12"; + case MachOTLVPage21: + return "MachOTLVPage21"; + case MachOTLVPageOffset12: + return "MachOTLVPageOffset12"; + case MachOPointerToGOT: + return "MachOPointerToGOT"; + case MachOPairedAddend: + return "MachOPairedAddend"; + case MachOLDRLiteral19: + return "MachOLDRLiteral19"; + case MachODelta32: + return "MachODelta32"; + case MachODelta64: + return "MachODelta64"; + case MachONegDelta32: + return "MachONegDelta32"; + case MachONegDelta64: + return "MachONegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); + } + } + unsigned NumSymbols = 0; }; @@ -430,32 +514,36 @@ class PerGraphGOTAndPLTStubsBuilder_MachO_arm64 PerGraphGOTAndPLTStubsBuilder_MachO_arm64>::PerGraphGOTAndPLTStubsBuilder; bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || - E.getKind() == TLVPage21 || E.getKind() == TLVPageOffset12 || - E.getKind() == PointerToGOT; + return E.getKind() == aarch64::GOTPage21 || + E.getKind() == aarch64::GOTPageOffset12 || + E.getKind() == aarch64::TLVPage21 || + E.getKind() == aarch64::TLVPageOffset12 || + E.getKind() == aarch64::PointerToGOT; } Symbol &createGOTEntry(Symbol &Target) { auto &GOTEntryBlock = G.createContentBlock( getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); - GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - if (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || - E.getKind() == TLVPage21 || E.getKind() == TLVPageOffset12) { + if (E.getKind() == aarch64::GOTPage21 || + E.getKind() == aarch64::GOTPageOffset12 || + E.getKind() == aarch64::TLVPage21 || + E.getKind() == aarch64::TLVPageOffset12) { // Update the target, but leave the edge addend as-is. E.setTarget(GOTEntry); - } else if (E.getKind() == PointerToGOT) { + } else if (E.getKind() == aarch64::PointerToGOT) { E.setTarget(GOTEntry); - E.setKind(Delta32); + E.setKind(aarch64::Delta32); } else llvm_unreachable("Not a GOT edge?"); } bool isExternalBranchEdge(Edge &E) { - return E.getKind() == Branch26 && !E.getTarget().isDefined(); + return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined(); } Symbol &createPLTStub(Symbol &Target) { @@ -463,12 +551,12 @@ class PerGraphGOTAndPLTStubsBuilder_MachO_arm64 getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); // Re-use GOT entries for stub targets. auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); + StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); } void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == Branch26 && "Not a Branch32 edge?"); + assert(E.getKind() == aarch64::Branch26 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); E.setTarget(Stub); } @@ -525,165 +613,8 @@ class MachOJITLinker_arm64 : public JITLinker { : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - - static unsigned getPageOffset12Shift(uint32_t Instr) { - constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; - constexpr uint32_t Vec128Mask = 0x04800000; - - if ((Instr & LoadStoreImm12Mask) == 0x39000000) { - uint32_t ImplicitShift = Instr >> 30; - if (ImplicitShift == 0) - if ((Instr & Vec128Mask) == Vec128Mask) - ImplicitShift = 4; - - return ImplicitShift; - } - - return 0; - } - Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { - using namespace support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Branch-inst is not 32-bit aligned"); - - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Branch26 target is not 32-bit " - "aligned"); - - if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - if (Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case Page21: - case TLVPage21: - case GOTPage21: { - assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && - "GOTPAGE21 with non-zero addend"); - uint64_t TargetPage = - (E.getTarget().getAddress().getValue() + E.getAddend()) & - ~static_cast(4096 - 1); - uint64_t PCPage = - FixupAddress.getValue() & ~static_cast(4096 - 1); - - int64_t PageDelta = TargetPage - PCPage; - if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xffffffe0) == 0x90000000 && - "RawInstr isn't an ADRP instruction"); - uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; - uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; - uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case PageOffset12: { - uint64_t TargetOffset = - (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - unsigned ImmShift = getPageOffset12Shift(RawInstr); - - if (TargetOffset & ((1 << ImmShift) - 1)) - return make_error("PAGEOFF12 target is not aligned"); - - uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case TLVPageOffset12: - case GOTPageOffset12: { - assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xfffffc00) == 0xf9400000 && - "RawInstr isn't a 64-bit LDR immediate"); - - uint32_t TargetOffset = E.getTarget().getAddress().getValue() & 0xfff; - assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); - uint32_t EncodedImm = (TargetOffset >> 3) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case LDRLiteral19: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "LDR is not 32-bit aligned"); - assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); - int64_t Delta = E.getTarget().getAddress() - FixupAddress; - if (Delta & 0x3) - return make_error("LDR literal target is not 32-bit " - "aligned"); - if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t EncodedImm = - ((static_cast(Delta) >> 2) & 0x7ffff) << 5; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - default: - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "unsupported edge kind" + - getMachOARM64RelocationKindName(E.getKind())); - } - - return Error::success(); + return aarch64::applyFixup(G, B, E); } uint64_t NullValue = 0; @@ -718,9 +649,9 @@ void link_MachO_arm64(std::unique_ptr G, // we support compact-unwind registration with libunwind. Config.PrePrunePasses.push_back( DWARFRecordSectionSplitter("__TEXT,__eh_frame")); - Config.PrePrunePasses.push_back( - EHFrameEdgeFixer("__TEXT,__eh_frame", 8, Pointer32, Pointer64, Delta32, - Delta64, NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + "__TEXT,__eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, + aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); // Add an in-place GOT/Stubs pass. Config.PostPrunePasses.push_back( @@ -734,44 +665,5 @@ void link_MachO_arm64(std::unique_ptr G, MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); } -const char *getMachOARM64RelocationKindName(Edge::Kind R) { - switch (R) { - case Branch26: - return "Branch26"; - case Pointer64: - return "Pointer64"; - case Pointer64Anon: - return "Pointer64Anon"; - case Page21: - return "Page21"; - case PageOffset12: - return "PageOffset12"; - case GOTPage21: - return "GOTPage21"; - case GOTPageOffset12: - return "GOTPageOffset12"; - case TLVPage21: - return "TLVPage21"; - case TLVPageOffset12: - return "TLVPageOffset12"; - case PointerToGOT: - return "PointerToGOT"; - case PairedAddend: - return "PairedAddend"; - case LDRLiteral19: - return "LDRLiteral19"; - case Delta32: - return "Delta32"; - case Delta64: - return "Delta64"; - case NegDelta32: - return "NegDelta32"; - case NegDelta64: - return "NegDelta64"; - default: - return getGenericEdgeKindName(static_cast(R)); - } -} - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 6dccc481188513..5a1aaee512e559 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,13 +18,202 @@ namespace llvm { namespace jitlink { namespace aarch64 { -const char *getEdgeKindName(Edge::Kind K) { - switch (K) { - case R_AARCH64_CALL26: - return "R_AARCH64_CALL26"; +unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + constexpr uint32_t Vec128Mask = 0x04800000; + + if ((Instr & LoadStoreImm12Mask) == 0x39000000) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; +} + +Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast(Value) & 0x3) + return make_error("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: + case TLVPage21: + case GOTPage21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; } - return getGenericEdgeKindName(K); + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case TLVPageOffset12: + case GOTPageOffset12: { + assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + + uint32_t TargetOffset = E.getTarget().getAddress().getValue() & 0xfff; + assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); + uint32_t EncodedImm = (TargetOffset >> 3) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); + } + + return Error::success(); } + +const char *getEdgeKindName(Edge::Kind R) { + switch (R) { + case Branch26: + return "Branch26"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case Page21: + return "Page21"; + case PageOffset12: + return "PageOffset12"; + case GOTPage21: + return "GOTPage21"; + case GOTPageOffset12: + return "GOTPageOffset12"; + case TLVPage21: + return "TLVPage21"; + case TLVPageOffset12: + return "TLVPageOffset12"; + case PointerToGOT: + return "PointerToGOT"; + case PairedAddend: + return "PairedAddend"; + case LDRLiteral19: + return "LDRLiteral19"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); + } +} + } // namespace aarch64 } // namespace jitlink } // namespace llvm From e230a2acbc907df6d2fa492151c86d83fa2cb030 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 7 Jun 2022 12:29:12 -0700 Subject: [PATCH 19/34] [JITLink][AArch64] Refactor isLoadStoreImm12 check out of getPageOffset12Shift. The separate isLoadStoreImm12 predicate will be used for validating ELF/aarch64 ldst relocation types. Reviewed By: lhames, sgraenitz Differential Revision: https://reviews.llvm.org/D126628 --- llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h | 2 ++ llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index ef38a1690c9b2b..5662c433d96a8e 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -43,6 +43,8 @@ enum EdgeKind_aarch64 : Edge::Kind { /// only const char *getEdgeKindName(Edge::Kind K); +bool isLoadStoreImm12(uint32_t Instr); + unsigned getPageOffset12Shift(uint32_t Instr); Error applyFixup(LinkGraph &G, Block &B, const Edge &E); diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 5a1aaee512e559..a4878f38bb4bdb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,11 +18,15 @@ namespace llvm { namespace jitlink { namespace aarch64 { -unsigned getPageOffset12Shift(uint32_t Instr) { +bool isLoadStoreImm12(uint32_t Instr) { constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + return (Instr & LoadStoreImm12Mask) == 0x39000000; +} + +unsigned getPageOffset12Shift(uint32_t Instr) { constexpr uint32_t Vec128Mask = 0x04800000; - if ((Instr & LoadStoreImm12Mask) == 0x39000000) { + if (isLoadStoreImm12(Instr)) { uint32_t ImplicitShift = Instr >> 30; if (ImplicitShift == 0) if ((Instr & Vec128Mask) == Vec128Mask) From 649e779967eca8c1dacfe78d1adb2c32e80c2c12 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 7 Jun 2022 14:03:52 -0700 Subject: [PATCH 20/34] [JITLink][ELF][AArch64] Implement ADR_PREL_PG_HI21, ADD_ABS_LO12_NC. Implements R_AARCH64_ADR_PREL_PG_HI21 and R_AARCH64_ADD_ABS_LO12_NC fixup edges using the generic aarch64 patch edges. Reviewed By: lhames, sgraenitz Differential Revision: https://reviews.llvm.org/D126287 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 28 +++++++-- .../JITLink/AArch64/ELF_aarch64_relocations.s | 57 +++++++++++++++++++ 2 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 1d00dab8212c31..008358d661d4a5 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -45,7 +45,9 @@ template class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { private: enum ELFAArch64RelocationKind : Edge::Kind { - ELFBranch26 = Edge::FirstRelocation, + ELFCall26 = Edge::FirstRelocation, + ELFAdrPage21, + ELFAddAbs12, }; static Expected @@ -53,7 +55,11 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { using namespace aarch64; switch (Type) { case ELF::R_AARCH64_CALL26: - return ELFBranch26; + return ELFCall26; + case ELF::R_AARCH64_ADR_PREL_PG_HI21: + return ELFAdrPage21; + case ELF::R_AARCH64_ADD_ABS_LO12_NC: + return ELFAddAbs12; } return make_error("Unsupported aarch64 relocation:" + @@ -105,10 +111,18 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Edge::Kind Kind = Edge::Invalid; switch (*RelocKind) { - case ELFBranch26: { + case ELFCall26: { Kind = aarch64::Branch26; break; } + case ELFAdrPage21: { + Kind = aarch64::Page21; + break; + } + case ELFAddAbs12: { + Kind = aarch64::PageOffset12; + break; + } }; Edge GE(Kind, Offset, *GraphSymbol, Addend); @@ -125,8 +139,12 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { /// Return the string name of the given ELF aarch64 edge kind. const char *getELFAArch64RelocationKindName(Edge::Kind R) { switch (R) { - case ELFBranch26: - return "ELFBranch26"; + case ELFCall26: + return "ELFCall26"; + case ELFAdrPage21: + return "ELFAdrPage21"; + case ELFAddAbs12: + return "ELFAddAbs12"; default: return getGenericEdgeKindName(static_cast(R)); } diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s new file mode 100644 index 00000000000000..5534fa9b769db0 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -0,0 +1,57 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false -position-independent -filetype=obj -o %t/elf_reloc.o %s +# RUN: llvm-jitlink -noexec -check %s %t/elf_reloc.o + + .text + + .globl main + .p2align 2 + .type main,@function +main: + ret + + .size main, .-main + +# Check R_AARCH64_CALL26 relocation of a local function call +# +# jitlink-check: decode_operand(local_func_call26, 0)[25:0] = (local_func - local_func_call26)[27:2] + .globl local_func + .p2align 2 + .type local_func,@function +local_func: + ret + .size local_func, .-local_func + + .globl local_func_call26 + .p2align 2 +local_func_call26: + bl local_func + .size local_func_call26, .-local_func_call26 + + +# Check R_AARCH64_ADR_PREL_PG_HI21 / R_AARCH64_ADD_ABS_LO12_NC relocation of a local symbol +# +# For the ADR_PREL_PG_HI21/ADRP instruction we have the 21-bit delta to the 4k page +# containing the global. +# +# jitlink-check: decode_operand(test_adr_prel, 1) = (named_data - test_adr_prel)[32:12] +# jitlink-check: decode_operand(test_add_abs_lo12, 2) = (named_data + 0)[11:0] + .globl test_adr_prel + .p2align 2 +test_adr_prel: + adrp x0, named_data + .size test_adr_prel, .-test_adr_prel + + .globl test_add_abs_lo12 + .p2align 2 +test_add_abs_lo12: + add x0, x0, :lo12:named_data + .size test_add_abs_lo12, .-test_add_abs_lo12 + + .globl named_data + .p2align 4 + .type named_data,@object +named_data: + .quad 0x2222222222222222 + .quad 0x3333333333333333 + .size named_data, .-named_data From f3d818fb1f6bbcd10a70d703147a58cc2f84f187 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 7 Jun 2022 17:16:41 -0700 Subject: [PATCH 21/34] [JITLink][ELF][AArch64] Implement R_AARCH64_LDST*_ABS_LO12_NC relocation types. Implement R_AARCH64_LDST*_ABS_LO12_NC relocaiton entries by reusing PageOffset21 generic relocation edge. The difference between MachO backend is that in ELF, the shift value is explicitly given by relocation type. lld generates the relocation type that matches with instruction bitwidth, so getting the shift value implicitly from instruction bytes should be fine in typical use cases. --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 86 +++++++++++++++++++ .../JITLink/AArch64/ELF_aarch64_relocations.s | 65 ++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 008358d661d4a5..5a24a69f9fd088 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -16,6 +16,7 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/MathExtras.h" #define DEBUG_TYPE "jitlink" @@ -48,6 +49,11 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { ELFCall26 = Edge::FirstRelocation, ELFAdrPage21, ELFAddAbs12, + ELFLdSt8Abs12, + ELFLdSt16Abs12, + ELFLdSt32Abs12, + ELFLdSt64Abs12, + ELFLdSt128Abs12, }; static Expected @@ -60,6 +66,16 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFAdrPage21; case ELF::R_AARCH64_ADD_ABS_LO12_NC: return ELFAddAbs12; + case ELF::R_AARCH64_LDST8_ABS_LO12_NC: + return ELFLdSt8Abs12; + case ELF::R_AARCH64_LDST16_ABS_LO12_NC: + return ELFLdSt16Abs12; + case ELF::R_AARCH64_LDST32_ABS_LO12_NC: + return ELFLdSt32Abs12; + case ELF::R_AARCH64_LDST64_ABS_LO12_NC: + return ELFLdSt64Abs12; + case ELF::R_AARCH64_LDST128_ABS_LO12_NC: + return ELFLdSt128Abs12; } return make_error("Unsupported aarch64 relocation:" + @@ -82,6 +98,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Error addSingleRelocation(const typename ELFT::Rela &Rel, const typename ELFT::Shdr &FixupSect, Block &BlockToFix) { + using support::ulittle32_t; using Base = ELFLinkGraphBuilder; uint32_t SymbolIndex = Rel.getSymbol(false); @@ -108,6 +125,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); + // Get a pointer to the fixup content. + const void *FixupContent = BlockToFix.getContent().data() + + (FixupAddress - BlockToFix.getAddress()); + Edge::Kind Kind = Edge::Invalid; switch (*RelocKind) { @@ -123,6 +144,61 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Kind = aarch64::PageOffset12; break; } + case ELFLdSt8Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 0) + return make_error( + "R_AARCH64_LDST8_ABS_LO12_NC target is not a " + "LDRB/STRB (imm12) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt16Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 1) + return make_error( + "R_AARCH64_LDST16_ABS_LO12_NC target is not a " + "LDRH/STRH (imm12) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt32Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 2) + return make_error( + "R_AARCH64_LDST32_ABS_LO12_NC target is not a " + "LDR/STR (imm12, 32 bit) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt64Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 3) + return make_error( + "R_AARCH64_LDST64_ABS_LO12_NC target is not a " + "LDR/STR (imm12, 64 bit) instruction"); + + Kind = aarch64::PageOffset12; + break; + } + case ELFLdSt128Abs12: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isLoadStoreImm12(Instr) || + aarch64::getPageOffset12Shift(Instr) != 4) + return make_error( + "R_AARCH64_LDST128_ABS_LO12_NC target is not a " + "LDR/STR (imm12, 128 bit) instruction"); + + Kind = aarch64::PageOffset12; + break; + } }; Edge GE(Kind, Offset, *GraphSymbol, Addend); @@ -145,6 +221,16 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return "ELFAdrPage21"; case ELFAddAbs12: return "ELFAddAbs12"; + case ELFLdSt8Abs12: + return "ELFLdSt8Abs12"; + case ELFLdSt16Abs12: + return "ELFLdSt16Abs12"; + case ELFLdSt32Abs12: + return "ELFLdSt32Abs12"; + case ELFLdSt64Abs12: + return "ELFLdSt64Abs12"; + case ELFLdSt128Abs12: + return "ELFLdSt128Abs12"; default: return getGenericEdgeKindName(static_cast(R)); } diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s index 5534fa9b769db0..1a670c2648a8a2 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -48,6 +48,71 @@ test_add_abs_lo12: add x0, x0, :lo12:named_data .size test_add_abs_lo12, .-test_add_abs_lo12 +# Check R_AARCH64_LDST*_ABS_LO12_NC relocation of a local symbol +# +# The immediate value should be the symbol address right shifted according to its instruction bitwidth. +# +# jitlink-check: decode_operand(test_ldrb, 2) = named_data[11:0] +# jitlink-check: decode_operand(test_ldrsb, 2) = (named_data + 0)[11:0] +# jitlink-check: decode_operand(test_ldrh, 2) = (named_data + 0)[11:1] +# jitlink-check: decode_operand(test_ldrsh, 2) = (named_data + 0)[11:1] +# jitlink-check: decode_operand(test_ldr_32bit, 2) = (named_data + 0)[11:2] +# jitlink-check: decode_operand(test_ldr_64bit, 2) = (named_data + 0)[11:3] +# jitlink-check: decode_operand(test_strb, 2) = named_data[11:0] +# jitlink-check: decode_operand(test_strh, 2) = (named_data + 0)[11:1] +# jitlink-check: decode_operand(test_str_32bit, 2) = (named_data + 0)[11:2] +# jitlink-check: decode_operand(test_str_64bit, 2) = (named_data + 0)[11:3] + + .globl test_ldrb +test_ldrb: + ldrb w0, [x1, :lo12:named_data] + .size test_ldrb, .-test_ldrb + + .globl test_ldrsb +test_ldrsb: + ldrsb w0, [x1, :lo12:named_data] + .size test_ldrsb, .-test_ldrsb + + .globl test_ldrh +test_ldrh: + ldrh w0, [x1, :lo12:named_data] + .size test_ldrh, .-test_ldrh + + .globl test_ldrsh +test_ldrsh: + ldrsh w0, [x1, :lo12:named_data] + .size test_ldrsh, .-test_ldrsh + + .globl test_ldr_32bit +test_ldr_32bit: + ldr w0, [x1, :lo12:named_data] + .size test_ldr_32bit, .-test_ldr_32bit + + .globl test_ldr_64bit +test_ldr_64bit: + ldr x0, [x1, :lo12:named_data] + .size test_ldr_64bit, .-test_ldr_64bit + + .globl test_strb +test_strb: + strb w0, [x1, :lo12:named_data] + .size test_strb, .-test_strb + + .globl test_strh +test_strh: + strh w0, [x1, :lo12:named_data] + .size test_strh, .-test_strh + + .globl test_str_32bit +test_str_32bit: + str w0, [x1, :lo12:named_data] + .size test_str_32bit, .-test_str_32bit + + .globl test_str_64bit +test_str_64bit: + str x0, [x1, :lo12:named_data] + .size test_str_64bit, .-test_str_64bit + .globl named_data .p2align 4 .type named_data,@object From 152c98d88afa6f1af2a91c7ca36de4cb4bbb12ce Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Tue, 7 Jun 2022 17:48:09 -0700 Subject: [PATCH 22/34] [JITLink][ELF][AArch64] Implement R_AARCH64_ABS64 relocation type. Implement R_AARCH64_ABS64 relocation entry. This relocation type is generated when creating a static function pointer to symbol. Reviewed By: lhames, sgraenitz Differential Revision: https://reviews.llvm.org/D126658 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 9 +++++++++ .../JITLink/AArch64/ELF_aarch64_relocations.s | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 5a24a69f9fd088..8ce1f67d63cf83 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -54,6 +54,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { ELFLdSt32Abs12, ELFLdSt64Abs12, ELFLdSt128Abs12, + ELFAbs64, }; static Expected @@ -76,6 +77,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFLdSt64Abs12; case ELF::R_AARCH64_LDST128_ABS_LO12_NC: return ELFLdSt128Abs12; + case ELF::R_AARCH64_ABS64: + return ELFAbs64; } return make_error("Unsupported aarch64 relocation:" + @@ -199,6 +202,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Kind = aarch64::PageOffset12; break; } + case ELFAbs64: { + Kind = aarch64::Pointer64; + break; + } }; Edge GE(Kind, Offset, *GraphSymbol, Addend); @@ -231,6 +238,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return "ELFLdSt64Abs12"; case ELFLdSt128Abs12: return "ELFLdSt128Abs12"; + case ELFAbs64: + return "ELFAbs64"; default: return getGenericEdgeKindName(static_cast(R)); } diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s index 1a670c2648a8a2..7a433e1715ba75 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -1,5 +1,6 @@ # RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false -position-independent -filetype=obj -o %t/elf_reloc.o %s +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false \ +# RUN: -position-independent -filetype=obj -o %t/elf_reloc.o %s # RUN: llvm-jitlink -noexec -check %s %t/elf_reloc.o .text @@ -113,6 +114,15 @@ test_str_64bit: str x0, [x1, :lo12:named_data] .size test_str_64bit, .-test_str_64bit +# Check R_AARCH64_ABS64 relocation of a function pointer to local symbol +# +# jitlink-check: *{8}local_func_addr_quad = named_func + .globl local_func_addr_quad + .p2align 3 +local_func_addr_quad: + .xword named_func + .size local_func_addr_quad, 8 + .globl named_data .p2align 4 .type named_data,@object @@ -120,3 +130,10 @@ named_data: .quad 0x2222222222222222 .quad 0x3333333333333333 .size named_data, .-named_data + + .globl named_func + .p2align 2 + .type named_func,@function +named_func: + ret + .size named_func, .-named_func From ce401b5c75ef4bb6d6968ed0ffe7d8e3e575b168 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 7 Jun 2022 18:17:26 -0700 Subject: [PATCH 23/34] [JITLink][ELF][AArch64] Implement ADR_GOT_PAGE, LD64_GOT_LO12_NC. This patch implements two most commonly used Global Offset Table relocations in ELF/AARCH64: R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC. It implements the GOT table manager by extending the existing PerGraphGOTAndPLTStubsBuilder. A future patch will unify this with the MachO implementation to produce a generic aarch64 got table manager. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127057 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 82 +++++++++++++++++++ .../JITLink/AArch64/ELF_aarch64_relocations.s | 32 +++++++- 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 8ce1f67d63cf83..02615a502250e4 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -19,6 +19,8 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/MathExtras.h" +#include "PerGraphGOTAndPLTStubsBuilder.h" + #define DEBUG_TYPE "jitlink" using namespace llvm; @@ -55,6 +57,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { ELFLdSt64Abs12, ELFLdSt128Abs12, ELFAbs64, + ELFAdrGOTPage21, + ELFLd64GOTLo12, }; static Expected @@ -79,6 +83,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFLdSt128Abs12; case ELF::R_AARCH64_ABS64: return ELFAbs64; + case ELF::R_AARCH64_ADR_GOT_PAGE: + return ELFAdrGOTPage21; + case ELF::R_AARCH64_LD64_GOT_LO12_NC: + return ELFLd64GOTLo12; } return make_error("Unsupported aarch64 relocation:" + @@ -206,6 +214,14 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Kind = aarch64::Pointer64; break; } + case ELFAdrGOTPage21: { + Kind = aarch64::GOTPage21; + break; + } + case ELFLd64GOTLo12: { + Kind = aarch64::GOTPageOffset12; + break; + } }; Edge GE(Kind, Offset, *GraphSymbol, Addend); @@ -240,6 +256,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return "ELFLdSt128Abs12"; case ELFAbs64: return "ELFAbs64"; + case ELFAdrGOTPage21: + return "ELFAdrGOTPage21"; + case ELFLd64GOTLo12: + return "ELFLd64GOTLo12"; default: return getGenericEdgeKindName(static_cast(R)); } @@ -252,6 +272,64 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { aarch64::getEdgeKindName) {} }; +class PerGraphGOTAndPLTStubsBuilder_ELF_arm64 + : public PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_ELF_arm64> { +public: + using PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_ELF_arm64>::PerGraphGOTAndPLTStubsBuilder; + + bool isGOTEdgeToFix(Edge &E) const { + return E.getKind() == aarch64::GOTPage21 || + E.getKind() == aarch64::GOTPageOffset12; + } + + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); + GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { + if (E.getKind() == aarch64::GOTPage21) { + E.setKind(aarch64::Page21); + E.setTarget(GOTEntry); + } else if (E.getKind() == aarch64::GOTPageOffset12) { + E.setKind(aarch64::PageOffset12); + E.setTarget(GOTEntry); + } else + llvm_unreachable("Not a GOT edge?"); + } + + bool isExternalBranchEdge(Edge &E) { return false; } + + Symbol &createPLTStub(Symbol &Target) { + assert(false && "unimplemetned"); + return Target; + } + + void fixPLTEdge(Edge &E, Symbol &Stub) { assert(false && "unimplemetned"); } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); + return *GOTSection; + } + + ArrayRef getGOTEntryBlockContent() { + return {reinterpret_cast(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; + } + + static const uint8_t NullGOTEntryContent[8]; + Section *GOTSection = nullptr; +}; + +const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::NullGOTEntryContent[8] = + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + Expected> createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { LLVM_DEBUG({ @@ -283,6 +361,10 @@ void link_ELF_aarch64(std::unique_ptr G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); } + + Config.PostPrunePasses.push_back( + PerGraphGOTAndPLTStubsBuilder_ELF_arm64::asPass); + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s index 7a433e1715ba75..2714ef4183f787 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -1,7 +1,9 @@ # RUN: rm -rf %t && mkdir -p %t # RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false \ # RUN: -position-independent -filetype=obj -o %t/elf_reloc.o %s -# RUN: llvm-jitlink -noexec -check %s %t/elf_reloc.o +# RUN: llvm-jitlink -noexec \ +# RUN: -abs external_data=0xdeadbeef \ +# RUN: -check %s %t/elf_reloc.o .text @@ -123,6 +125,34 @@ local_func_addr_quad: .xword named_func .size local_func_addr_quad, 8 +# Check R_AARCH64_ADR_GOT_PAGE / R_AARCH64_LD64_GOT_LO12_NC handling with a +# reference to an external symbol. Validate both the reference to the GOT entry, +# and also the content of the GOT entry. +# +# For the ADRP :got: instruction we have the 21-bit delta to the 4k page +# containing the GOT entry for external_data. +# +# For the LDR :got_lo12: instruction we have the 12-bit offset of the entry +# within the page. +# +# jitlink-check: *{8}(got_addr(elf_reloc.o, external_data)) = external_data +# jitlink-check: decode_operand(test_adr_gotpage_external, 1) = \ +# jitlink-check: (got_addr(elf_reloc.o, external_data)[32:12] - \ +# jitlink-check: test_adr_gotpage_external[32:12]) +# jitlink-check: decode_operand(test_ld64_gotlo12_external, 2) = \ +# jitlink-check: got_addr(elf_reloc.o, external_data)[11:3] + .globl test_adr_gotpage_external + .p2align 2 +test_adr_gotpage_external: + adrp x0, :got:external_data + .size test_adr_gotpage_external, .-test_adr_gotpage_external + + .globl test_ld64_gotlo12_external + .p2align 2 +test_ld64_gotlo12_external: + ldr x0, [x0, :got_lo12:external_data] + .size test_ld64_gotlo12_external, .-test_ld64_gotlo12_external + .globl named_data .p2align 4 .type named_data,@object From a84660c35660f7de945d494e0135b66e511b22bd Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Wed, 8 Jun 2022 11:13:03 -0700 Subject: [PATCH 24/34] [JITLink][ELF][AArch64] Implement R_AARCH64_PREL32 and R_AARCH64_PREL64. This patch implements R_AARCH64_PREL64 and R_AARCH64_PREL32 relocations that is used in eh frame pointers. The test case utlizes obj2yaml tool to create an artifical eh frame that generates related relocation types. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127058 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 18 ++++++ .../JITLink/AArch64/ELF_aarch64_ehframe.test | 60 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 02615a502250e4..7711e93f0e79dd 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -57,6 +57,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { ELFLdSt64Abs12, ELFLdSt128Abs12, ELFAbs64, + ELFPrel32, + ELFPrel64, ELFAdrGOTPage21, ELFLd64GOTLo12, }; @@ -83,6 +85,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFLdSt128Abs12; case ELF::R_AARCH64_ABS64: return ELFAbs64; + case ELF::R_AARCH64_PREL32: + return ELFPrel32; + case ELF::R_AARCH64_PREL64: + return ELFPrel64; case ELF::R_AARCH64_ADR_GOT_PAGE: return ELFAdrGOTPage21; case ELF::R_AARCH64_LD64_GOT_LO12_NC: @@ -214,6 +220,14 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Kind = aarch64::Pointer64; break; } + case ELFPrel32: { + Kind = aarch64::Delta32; + break; + } + case ELFPrel64: { + Kind = aarch64::Delta64; + break; + } case ELFAdrGOTPage21: { Kind = aarch64::GOTPage21; break; @@ -256,6 +270,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return "ELFLdSt128Abs12"; case ELFAbs64: return "ELFAbs64"; + case ELFPrel32: + return "ELFPrel32"; + case ELFPrel64: + return "ELFPrel64"; case ELFAdrGOTPage21: return "ELFAdrGOTPage21"; case ELFLd64GOTLo12: diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test new file mode 100644 index 00000000000000..fa224d856c4efa --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.test @@ -0,0 +1,60 @@ +# RUN: yaml2obj -o %t.o %s +# RUN: llvm-jitlink -noexec -check %s %t.o + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_AARCH64 + SectionHeaderStringTable: .strtab +Sections: + - Name: .text + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + AddressAlign: 0x4 + Content: E0031F2AC0035FD6 + - Name: .eh_frame + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + AddressAlign: 0x8 + Content: 1000000000000000017A5200017C1E011B0C1F001000000018000000000000000800000000000000 + - Name: .rela.eh_frame + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .eh_frame + Relocations: + - Offset: 0x1C + Symbol: text + Type: R_AARCH64_PREL32 + - Offset: 0x20 + Symbol: text + Type: R_AARCH64_PREL64 + - Type: SectionHeaderTable + Sections: + - Name: .strtab + - Name: .text + - Name: .eh_frame + - Name: .rela.eh_frame + - Name: .symtab +Symbols: + - Name: text + Type: STT_SECTION + Binding: STB_GLOBAL + Section: .text + Size: 0x8 + - Name: eh_frame + Type: STT_SECTION + Binding: STB_GLOBAL + Section: .eh_frame + Size: 0x28 + - Name: main + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Size: 0x8 + +# jitlink-check: *{4}(eh_frame + 28) = (text + 0) - (eh_frame + 28) +# jitlink-check: *{8}(eh_frame + 32) = (text + 0) - (eh_frame + 32) \ No newline at end of file From 11aab972726b786eb4c14e3c36e49596c570300d Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Fri, 10 Jun 2022 08:41:33 +0900 Subject: [PATCH 25/34] [JITLink][AArch64] Fix overflow range of Page21 fixup edge. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allowed range for Page21 relocation is -2^32 <= X < 2^32 in both ELF and MachO. https://github.com/llvm/llvm-project/blob/09c2b7c35af8c4bad39f03e9f60df8bd07323028/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h#L210 (MachO) ELF for the ARM ® 64-bit Architecture (AArch64) Table 4-9 (ELF) Reviewed By: sgraenitz Differential Revision: https://reviews.llvm.org/D126387 --- llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index a4878f38bb4bdb..fb5429eed96e34 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -92,7 +92,7 @@ Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { FixupAddress.getValue() & ~static_cast(4096 - 1); int64_t PageDelta = TargetPage - PCPage; - if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) + if (!isInt<33>(PageDelta)) return makeTargetOutOfRangeError(G, B, E); uint32_t RawInstr = *(ulittle32_t *)FixupPtr; From ce6c028dcad9ffea9a0d1c40150ef5e61d1b03f5 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Fri, 10 Jun 2022 10:57:16 +0900 Subject: [PATCH 26/34] [JITLink][ELF][AArch64] Implement R_AARCH64_JUMP26 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements R_AARCH64_JUMP26. We can use the same generic aarch64 Branch26 edge since B instruction and BL nstruction have the same sized&offseted immediate field, and the relocation address calculation is the same. Reference: ELF for the ARM ® 64-bit Architecture Tabel 4-10, ARM Architecture Reference Manual ® ARMv8, for ARMv8-A architecture profile C6.2.24, C6.2.31 Reviewed By: sgraenitz Differential Revision: https://reviews.llvm.org/D127059 --- llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp | 1 + .../JITLink/AArch64/ELF_aarch64_relocations.s | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 7711e93f0e79dd..ae9963735c612e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -68,6 +68,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { using namespace aarch64; switch (Type) { case ELF::R_AARCH64_CALL26: + case ELF::R_AARCH64_JUMP26: return ELFCall26; case ELF::R_AARCH64_ADR_PREL_PG_HI21: return ELFAdrPage21; diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s index 2714ef4183f787..86048783432b0a 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -15,9 +15,10 @@ main: .size main, .-main -# Check R_AARCH64_CALL26 relocation of a local function call +# Check R_AARCH64_CALL26 / R_AARCH64_JUMP26 relocation of a local function call # # jitlink-check: decode_operand(local_func_call26, 0)[25:0] = (local_func - local_func_call26)[27:2] +# jitlink-check: decode_operand(local_func_jump26, 0)[25:0] = (local_func - local_func_jump26)[27:2] .globl local_func .p2align 2 .type local_func,@function @@ -31,6 +32,11 @@ local_func_call26: bl local_func .size local_func_call26, .-local_func_call26 + .globl local_func_jump26 + .p2align 2 +local_func_jump26: + b local_func + .size local_func_jump26, .-local_func_jump26 # Check R_AARCH64_ADR_PREL_PG_HI21 / R_AARCH64_ADD_ABS_LO12_NC relocation of a local symbol # From 3bee8c9277b6f8c6f5e96a994fd55ca82884dc41 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Fri, 10 Jun 2022 13:46:31 +0900 Subject: [PATCH 27/34] [JITLink][ELF][AArch64] Implement Procedure Linkage Table. Implements Procedure Linkage Table (PLT) for ELF/AARCH64. The aarch64 linux calling convention also uses r16 as the intra-procedure-call scratch register same as MachO/ARM64. We can use the same stub sequence for this reason. Also, BR regiseter doesn't touch X30 register. External function call by BL instruction (touched by CALL26 relocation) will set X30 to the original PC + 4, which is the intended behavior. External function call by B instruction (touched by JUMP26 relocation) doesn't requite to set X30, so the patch will be correct in this case too. Reference: https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#611general-purpose-registers Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127061 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 35 ++++++++++++++++--- .../JITLink/AArch64/ELF_aarch64_relocations.s | 28 +++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index ae9963735c612e..0f6f73192e175a 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -321,14 +321,24 @@ class PerGraphGOTAndPLTStubsBuilder_ELF_arm64 llvm_unreachable("Not a GOT edge?"); } - bool isExternalBranchEdge(Edge &E) { return false; } + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined(); + } Symbol &createPLTStub(Symbol &Target) { - assert(false && "unimplemetned"); - return Target; + auto &StubContentBlock = G.createContentBlock( + getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = getGOTEntry(Target); + StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); } - void fixPLTEdge(Edge &E, Symbol &Stub) { assert(false && "unimplemetned"); } + void fixPLTEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == aarch64::Branch26 && "Not a Branch26 edge?"); + assert(E.getAddend() == 0 && "Branch26 edge has non-zero addend?"); + E.setTarget(Stub); + } private: Section &getGOTSection() { @@ -337,17 +347,34 @@ class PerGraphGOTAndPLTStubsBuilder_ELF_arm64 return *GOTSection; } + Section &getStubsSection() { + if (!StubsSection) + StubsSection = + &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); + return *StubsSection; + } + ArrayRef getGOTEntryBlockContent() { return {reinterpret_cast(NullGOTEntryContent), sizeof(NullGOTEntryContent)}; } + ArrayRef getStubBlockContent() { + return {reinterpret_cast(StubContent), sizeof(StubContent)}; + } + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[8]; Section *GOTSection = nullptr; + Section *StubsSection = nullptr; }; const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::StubContent[8] = { + 0x10, 0x00, 0x00, 0x58, // LDR x16, + 0x00, 0x02, 0x1f, 0xd6 // BR x16 +}; Expected> createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s index 86048783432b0a..8714f09095c830 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -3,6 +3,7 @@ # RUN: -position-independent -filetype=obj -o %t/elf_reloc.o %s # RUN: llvm-jitlink -noexec \ # RUN: -abs external_data=0xdeadbeef \ +# RUN: -abs external_func=0xcafef00d \ # RUN: -check %s %t/elf_reloc.o .text @@ -57,6 +58,24 @@ test_add_abs_lo12: add x0, x0, :lo12:named_data .size test_add_abs_lo12, .-test_add_abs_lo12 +# Check that calls/jumps to external functions trigger the generation of stubs and GOT +# entries. +# +# jitlink-check: decode_operand(test_external_call, 0) = (stub_addr(elf_reloc.o, external_func) - test_external_call)[27:2] +# jitlink-check: decode_operand(test_external_jump, 0) = (stub_addr(elf_reloc.o, external_func) - test_external_jump)[27:2] +# jitlink-check: *{8}(got_addr(elf_reloc.o, external_func)) = external_func + .globl test_external_call + .p2align 2 +test_external_call: + bl external_func + .size test_external_call, .-test_external_call + + .globl test_external_jump + .p2align 2 +test_external_jump: + b external_func + .size test_external_jump, .-test_external_jump + # Check R_AARCH64_LDST*_ABS_LO12_NC relocation of a local symbol # # The immediate value should be the symbol address right shifted according to its instruction bitwidth. @@ -131,6 +150,15 @@ local_func_addr_quad: .xword named_func .size local_func_addr_quad, 8 +# Check R_AARCH64_ABS64 relocation of a function pointer to external symbol +# +# jitlink-check: *{8}external_func_addr_quad = external_func + .globl external_func_addr_quad + .p2align 3 +external_func_addr_quad: + .xword external_func + .size external_func_addr_quad, 8 + # Check R_AARCH64_ADR_GOT_PAGE / R_AARCH64_LD64_GOT_LO12_NC handling with a # reference to an external symbol. Validate both the reference to the GOT entry, # and also the content of the GOT entry. From 176d08843dbb188361b1ddaf58ff12a0506292dc Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Fri, 10 Jun 2022 15:22:18 +0900 Subject: [PATCH 28/34] [JITLink][EHFrameSupport] Remove CodeAlignmentFactor and DataAlignmentFactor validation. Removes CodeAlignmentFactor and DataAlignmentFactor validation in EHFrameEdgeFixer. I observed some of aarch64 elf files generated by clang contains CIE record with code_alignment_factor = 4 or data_alignment_factor = -8. code_alignment_factor and data_alignment_factor are used by call fram instruction that should be correctled handled by libunwind. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127062 --- llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 8f5c990e003600..25a18d45e27b4d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -213,10 +213,6 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, uint64_t CodeAlignmentFactor = 0; if (auto Err = RecordReader.readULEB128(CodeAlignmentFactor)) return Err; - if (CodeAlignmentFactor != 1) - return make_error("Unsupported CIE code alignment factor " + - Twine(CodeAlignmentFactor) + - " (expected 1)"); } // Read and validate the data alignment factor. @@ -224,10 +220,6 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, int64_t DataAlignmentFactor = 0; if (auto Err = RecordReader.readSLEB128(DataAlignmentFactor)) return Err; - if (DataAlignmentFactor != -8) - return make_error("Unsupported CIE data alignment factor " + - Twine(DataAlignmentFactor) + - " (expected -8)"); } // Skip the return address register field. From cb01a7120bad8b2c67f812f0abe89fda8cef2b4e Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Sat, 11 Jun 2022 03:34:49 +0900 Subject: [PATCH 29/34] [JITLink][ELF][AArch64] Implement eh frame handling. Implements eh frame handling by using generic EHFrame passes. The c++ exception handling works correctly with this change. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127063 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 6 ++ .../JITLink/AArch64/ELF_aarch64_ehframe.s | 80 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 0f6f73192e175a..e6b44b11e91be0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -11,9 +11,11 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" +#include "EHFrameSupportImpl.h" #include "ELFLinkGraphBuilder.h" #include "JITLinkGeneric.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" @@ -402,6 +404,10 @@ void link_ELF_aarch64(std::unique_ptr G, PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, + aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s new file mode 100644 index 00000000000000..ff7e5a6546f6f4 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_ehframe.s @@ -0,0 +1,80 @@ +# REQUIRES: asserts +# RUN: llvm-mc -triple=aarch64-linux-gnu -filetype=obj -o %t %s +# RUN: llvm-jitlink -noexec -phony-externals -debug-only=jitlink %t 2>&1 | \ +# RUN: FileCheck %s +# +# Check that splitting of eh-frame sections works. +# +# CHECK: DWARFRecordSectionSplitter: Processing .eh_frame... +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Extracted {{.*}} section = .eh_frame +# CHECK: Processing CFI record at +# CHECK: Extracted {{.*}} section = .eh_frame +# CHECK: EHFrameEdgeFixer: Processing .eh_frame... +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Record is CIE +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Record is FDE +# CHECK: Adding edge at {{.*}} to CIE at: {{.*}} +# CHECK: Existing edge at {{.*}} to PC begin at {{.*}} +# CHECK: Adding keep-alive edge from target at {{.*}} to FDE at {{.*}} +# CHECK: Processing block at +# CHECK: Processing CFI record at +# CHECK: Record is FDE +# CHECK: Adding edge at {{.*}} to CIE at: {{.*}} +# CHECK: Existing edge at {{.*}} to PC begin at {{.*}} +# CHECK: Adding keep-alive edge from target at {{.*}} to FDE at {{.*}} + + .text + .globl main + .p2align 2 + .type main,@function +main: + .cfi_startproc + sub sp, sp, #32 + .cfi_def_cfa_offset 32 + stp x29, x30, [sp, #16] + add x29, sp, #16 + .cfi_def_cfa w29, 16 + .cfi_offset w30, -8 + .cfi_offset w29, -16 + stur wzr, [x29, #-4] + mov x0, #4 + bl __cxa_allocate_exception + mov w8, #1 + str w8, [x0] + adrp x1, :got:_ZTIi + ldr x1, [x1, :got_lo12:_ZTIi] + mov x2, xzr + bl __cxa_throw +.main_end: + .size main, .main_end-main + .cfi_endproc + + .globl dup + .p2align 2 + .type dup,@function +dup: + .cfi_startproc + sub sp, sp, #32 + .cfi_def_cfa_offset 32 + stp x29, x30, [sp, #16] + add x29, sp, #16 + .cfi_def_cfa w29, 16 + .cfi_offset w30, -8 + .cfi_offset w29, -16 + stur wzr, [x29, #-4] + mov x0, #4 + bl __cxa_allocate_exception + mov w8, #1 + str w8, [x0] + adrp x1, :got:_ZTIi + ldr x1, [x1, :got_lo12:_ZTIi] + mov x2, xzr + bl __cxa_throw +.dup_end: + .size dup, .dup_end-dup + .cfi_endproc \ No newline at end of file From c4d7e6ce7c73b5b1b87d2ff0122fb55452e1fd95 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 14 Jun 2022 10:58:05 +0900 Subject: [PATCH 30/34] [JITLink][AArch64] Unify table managers of ELF and MachO. Unifies GOT/PLT table managers of ELF and MachO on aarch64 architecture. Additionally, it migrates table managers from PerGraphGOTAndPLTStubsBuilder to generic crtp TableManager. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127558 --- .../llvm/ExecutionEngine/JITLink/aarch64.h | 121 ++++++++++++++++++ .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 105 +++------------ .../ExecutionEngine/JITLink/MachO_arm64.cpp | 105 ++------------- llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 36 +++--- 4 files changed, 166 insertions(+), 201 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 5662c433d96a8e..36403fab9efbb5 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -13,7 +13,9 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H #define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H +#include "TableManager.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" namespace llvm { namespace jitlink { @@ -49,6 +51,125 @@ unsigned getPageOffset12Shift(uint32_t Instr); Error applyFixup(LinkGraph &G, Block &B, const Edge &E); +/// AArch64 null pointer content. +extern const uint8_t NullGOTEntryContent[8]; + +/// AArch64 PLT stub content. +extern const uint8_t StubContent[8]; + +/// Global Offset Table Builder. +class GOTTableManager : public TableManager { +public: + static StringRef getSectionName() { return "$__GOT"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind KindToSet = Edge::Invalid; + const char *BlockWorkingMem = B->getContent().data(); + const char *FixupPtr = BlockWorkingMem + E.getOffset(); + + switch (E.getKind()) { + case aarch64::GOTPage21: + case aarch64::TLVPage21: { + KindToSet = aarch64::Page21; + break; + } + case aarch64::GOTPageOffset12: + case aarch64::TLVPageOffset12: { + KindToSet = aarch64::PageOffset12; + uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr; + assert(E.getAddend() == 0 && + "GOTPageOffset12/TLVPageOffset12 with non-zero addend"); + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + break; + } + case aarch64::PointerToGOT: { + KindToSet = aarch64::Delta64; + break; + } + default: + return false; + } + assert(KindToSet != Edge::Invalid && + "Fell through switch, but no new kind to set"); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(KindToSet); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(G), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); + GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + +private: + Section &getGOTSection(LinkGraph &G) { + if (!GOTSection) + GOTSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *GOTSection; + } + + ArrayRef getGOTEntryBlockContent() { + return {reinterpret_cast(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; + } + + Section *GOTSection = nullptr; +}; + +/// Procedure Linkage Table Builder. +class PLTTableManager : public TableManager { +public: + PLTTableManager(GOTTableManager &GOT) : GOT(GOT) {} + + static StringRef getSectionName() { return "$__STUBS"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined()) { + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + return false; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + auto &StubContentBlock = G.createContentBlock( + getStubsSection(G), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = GOT.getEntryForTarget(G, Target); + StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); + } + +public: + Section &getStubsSection(LinkGraph &G) { + if (!StubsSection) + StubsSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *StubsSection; + } + + ArrayRef getStubBlockContent() { + return {reinterpret_cast(StubContent), sizeof(StubContent)}; + } + + GOTTableManager &GOT; + Section *StubsSection = nullptr; +}; + } // namespace aarch64 } // namespace jitlink } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index e6b44b11e91be0..9c2dff77658287 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -21,15 +21,12 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/MathExtras.h" -#include "PerGraphGOTAndPLTStubsBuilder.h" - #define DEBUG_TYPE "jitlink" using namespace llvm; using namespace llvm::jitlink; -namespace llvm { -namespace jitlink { +namespace { class ELFJITLinker_aarch64 : public JITLinker { friend class JITLinker; @@ -293,90 +290,19 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { aarch64::getEdgeKindName) {} }; -class PerGraphGOTAndPLTStubsBuilder_ELF_arm64 - : public PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_ELF_arm64> { -public: - using PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_ELF_arm64>::PerGraphGOTAndPLTStubsBuilder; - - bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == aarch64::GOTPage21 || - E.getKind() == aarch64::GOTPageOffset12; - } - - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); - GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); - } - - void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - if (E.getKind() == aarch64::GOTPage21) { - E.setKind(aarch64::Page21); - E.setTarget(GOTEntry); - } else if (E.getKind() == aarch64::GOTPageOffset12) { - E.setKind(aarch64::PageOffset12); - E.setTarget(GOTEntry); - } else - llvm_unreachable("Not a GOT edge?"); - } - - bool isExternalBranchEdge(Edge &E) { - return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined(); - } - - Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = G.createContentBlock( - getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); - return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); - } - - void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == aarch64::Branch26 && "Not a Branch26 edge?"); - assert(E.getAddend() == 0 && "Branch26 edge has non-zero addend?"); - E.setTarget(Stub); - } - -private: - Section &getGOTSection() { - if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); - return *GOTSection; - } - - Section &getStubsSection() { - if (!StubsSection) - StubsSection = - &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); - return *StubsSection; - } - - ArrayRef getGOTEntryBlockContent() { - return {reinterpret_cast(NullGOTEntryContent), - sizeof(NullGOTEntryContent)}; - } +Error buildTables_ELF_aarch64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); - ArrayRef getStubBlockContent() { - return {reinterpret_cast(StubContent), sizeof(StubContent)}; - } + aarch64::GOTTableManager GOT; + aarch64::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[8]; - Section *GOTSection = nullptr; - Section *StubsSection = nullptr; -}; +} // namespace -const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::NullGOTEntryContent[8] = - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_arm64::StubContent[8] = { - 0x10, 0x00, 0x00, 0x58, // LDR x16, - 0x00, 0x02, 0x1f, 0xd6 // BR x16 -}; +namespace llvm { +namespace jitlink { Expected> createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { @@ -404,18 +330,21 @@ void link_ELF_aarch64(std::unique_ptr G, PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add eh-frame passses. Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); Config.PrePrunePasses.push_back(EHFrameEdgeFixer( ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); + + // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else Config.PrePrunePasses.push_back(markAllSymbolsLive); - } - Config.PostPrunePasses.push_back( - PerGraphGOTAndPLTStubsBuilder_ELF_arm64::asPass); + // Add an in-place GOT/Stubs build pass. + Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); + } if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 1356f53b629220..dd50314d3ed754 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -15,7 +15,6 @@ #include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "MachOLinkGraphBuilder.h" -#include "PerGraphGOTAndPLTStubsBuilder.h" #define DEBUG_TYPE "jitlink" @@ -506,103 +505,20 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { unsigned NumSymbols = 0; }; -class PerGraphGOTAndPLTStubsBuilder_MachO_arm64 - : public PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_MachO_arm64> { -public: - using PerGraphGOTAndPLTStubsBuilder< - PerGraphGOTAndPLTStubsBuilder_MachO_arm64>::PerGraphGOTAndPLTStubsBuilder; - - bool isGOTEdgeToFix(Edge &E) const { - return E.getKind() == aarch64::GOTPage21 || - E.getKind() == aarch64::GOTPageOffset12 || - E.getKind() == aarch64::TLVPage21 || - E.getKind() == aarch64::TLVPageOffset12 || - E.getKind() == aarch64::PointerToGOT; - } - - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); - GOTEntryBlock.addEdge(aarch64::Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); - } - - void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - if (E.getKind() == aarch64::GOTPage21 || - E.getKind() == aarch64::GOTPageOffset12 || - E.getKind() == aarch64::TLVPage21 || - E.getKind() == aarch64::TLVPageOffset12) { - // Update the target, but leave the edge addend as-is. - E.setTarget(GOTEntry); - } else if (E.getKind() == aarch64::PointerToGOT) { - E.setTarget(GOTEntry); - E.setKind(aarch64::Delta32); - } else - llvm_unreachable("Not a GOT edge?"); - } - - bool isExternalBranchEdge(Edge &E) { - return E.getKind() == aarch64::Branch26 && !E.getTarget().isDefined(); - } - - Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = G.createContentBlock( - getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); - StubContentBlock.addEdge(aarch64::LDRLiteral19, 0, GOTEntrySymbol, 0); - return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); - } - - void fixPLTEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == aarch64::Branch26 && "Not a Branch32 edge?"); - assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); - E.setTarget(Stub); - } - -private: - Section &getGOTSection() { - if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); - return *GOTSection; - } - - Section &getStubsSection() { - if (!StubsSection) - StubsSection = - &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); - return *StubsSection; - } - - ArrayRef getGOTEntryBlockContent() { - return {reinterpret_cast(NullGOTEntryContent), - sizeof(NullGOTEntryContent)}; - } - - ArrayRef getStubBlockContent() { - return {reinterpret_cast(StubContent), sizeof(StubContent)}; - } - - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[8]; - Section *GOTSection = nullptr; - Section *StubsSection = nullptr; -}; - -const uint8_t - PerGraphGOTAndPLTStubsBuilder_MachO_arm64::NullGOTEntryContent[8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_arm64::StubContent[8] = { - 0x10, 0x00, 0x00, 0x58, // LDR x16, - 0x00, 0x02, 0x1f, 0xd6 // BR x16 -}; - } // namespace namespace llvm { namespace jitlink { +Error buildTables_MachO_arm64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + aarch64::GOTTableManager GOT; + aarch64::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} + class MachOJITLinker_arm64 : public JITLinker { friend class JITLinker; @@ -654,8 +570,7 @@ void link_MachO_arm64(std::unique_ptr G, aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back( - PerGraphGOTAndPLTStubsBuilder_MachO_arm64::asPass); + Config.PostPrunePasses.push_back(buildTables_MachO_arm64); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index fb5429eed96e34..697c31b65a0b2b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -80,9 +80,7 @@ Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { *(ulittle64_t *)FixupPtr = Value; break; } - case Page21: - case TLVPage21: - case GOTPage21: { + case Page21: { assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && "GOTPAGE21 with non-zero addend"); uint64_t TargetPage = @@ -119,21 +117,6 @@ Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { *(ulittle32_t *)FixupPtr = FixedInstr; break; } - case TLVPageOffset12: - case GOTPageOffset12: { - assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xfffffc00) == 0xf9400000 && - "RawInstr isn't a 64-bit LDR immediate"); - - uint32_t TargetOffset = E.getTarget().getAddress().getValue() & 0xfff; - assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); - uint32_t EncodedImm = (TargetOffset >> 3) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } case LDRLiteral19: { assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); @@ -170,6 +153,15 @@ Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { *(little64_t *)FixupPtr = Value; break; } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } default: return make_error( "In graph " + G.getName() + ", section " + B.getSection().getName() + @@ -179,6 +171,14 @@ Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { return Error::success(); } +const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; + +const uint8_t StubContent[8] = { + 0x10, 0x00, 0x00, 0x58, // LDR x16, + 0x00, 0x02, 0x1f, 0xd6 // BR x16 +}; + const char *getEdgeKindName(Edge::Kind R) { switch (R) { case Branch26: From a68eebad9d21d1ae61d6ec5871a53bdf0cadd944 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 14 Jun 2022 10:58:06 +0900 Subject: [PATCH 31/34] [JITLink][AArch64] Lift fixup functions from aarch64.cpp to aarch64.h. (NFC) Lift fixup functions from aarch64.cpp to aarch64.h so that they have better chance of getting inlined. Also, adds some comments documenting the purpose of functions. Reviewed By: sgraenitz Differential Revision: https://reviews.llvm.org/D127559 --- .../llvm/ExecutionEngine/JITLink/aarch64.h | 161 +++++++++++++++++- llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 153 ----------------- 2 files changed, 158 insertions(+), 156 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 36403fab9efbb5..2cebc03310e171 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -45,11 +45,166 @@ enum EdgeKind_aarch64 : Edge::Kind { /// only const char *getEdgeKindName(Edge::Kind K); -bool isLoadStoreImm12(uint32_t Instr); +// Returns whether the Instr is LD/ST (imm12) +inline bool isLoadStoreImm12(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + return (Instr & LoadStoreImm12Mask) == 0x39000000; +} -unsigned getPageOffset12Shift(uint32_t Instr); +// Returns the amount the address operand of LD/ST (imm12) +// should be shifted right by. +// +// The shift value varies by the data size of LD/ST instruction. +// For instance, LDH instructoin needs the address to be shifted +// right by 1. +inline unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t Vec128Mask = 0x04800000; + + if (isLoadStoreImm12(Instr)) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; +} + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast(Value) & 0x3) + return make_error("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (!isInt<33>(PageDelta)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); + } -Error applyFixup(LinkGraph &G, Block &B, const Edge &E); + return Error::success(); +} /// AArch64 null pointer content. extern const uint8_t NullGOTEntryContent[8]; diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 697c31b65a0b2b..1397a45df4c2fb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,159 +18,6 @@ namespace llvm { namespace jitlink { namespace aarch64 { -bool isLoadStoreImm12(uint32_t Instr) { - constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; - return (Instr & LoadStoreImm12Mask) == 0x39000000; -} - -unsigned getPageOffset12Shift(uint32_t Instr) { - constexpr uint32_t Vec128Mask = 0x04800000; - - if (isLoadStoreImm12(Instr)) { - uint32_t ImplicitShift = Instr >> 30; - if (ImplicitShift == 0) - if ((Instr & Vec128Mask) == Vec128Mask) - ImplicitShift = 4; - - return ImplicitShift; - } - - return 0; -} - -Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { - using namespace support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Branch-inst is not 32-bit aligned"); - - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Branch26 target is not 32-bit " - "aligned"); - - if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - if (Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case Page21: { - assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && - "GOTPAGE21 with non-zero addend"); - uint64_t TargetPage = - (E.getTarget().getAddress().getValue() + E.getAddend()) & - ~static_cast(4096 - 1); - uint64_t PCPage = - FixupAddress.getValue() & ~static_cast(4096 - 1); - - int64_t PageDelta = TargetPage - PCPage; - if (!isInt<33>(PageDelta)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xffffffe0) == 0x90000000 && - "RawInstr isn't an ADRP instruction"); - uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; - uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; - uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case PageOffset12: { - uint64_t TargetOffset = - (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - unsigned ImmShift = getPageOffset12Shift(RawInstr); - - if (TargetOffset & ((1 << ImmShift) - 1)) - return make_error("PAGEOFF12 target is not aligned"); - - uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case LDRLiteral19: { - assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); - assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); - int64_t Delta = E.getTarget().getAddress() - FixupAddress; - if (Delta & 0x3) - return make_error("LDR literal target is not 32-bit " - "aligned"); - if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - case TLVPage21: - case GOTPage21: - case TLVPageOffset12: - case GOTPageOffset12: - case PointerToGOT: { - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); - } - default: - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "unsupported edge kind" + getEdgeKindName(E.getKind())); - } - - return Error::success(); -} - const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; From eee4a9da173acaefd99594fc6167eca71089e464 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 14 Jun 2022 13:50:47 +0900 Subject: [PATCH 32/34] [JITLink][AArch64] Implement MoveWide16 generic edge. Implements MoveWide16 generic edge kind that can be used to patch MOVZ/MOVK (imm16) instructions. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127584 --- .../llvm/ExecutionEngine/JITLink/aarch64.h | 34 +++++++++++++++++++ llvm/lib/ExecutionEngine/JITLink/aarch64.cpp | 2 ++ 2 files changed, 36 insertions(+) diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 2cebc03310e171..42a01c25de146b 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -28,6 +28,7 @@ enum EdgeKind_aarch64 : Edge::Kind { Pointer64Anon, Page21, PageOffset12, + MoveWide16, GOTPage21, GOTPageOffset12, TLVPage21, @@ -72,6 +73,25 @@ inline unsigned getPageOffset12Shift(uint32_t Instr) { return 0; } +// Returns whether the Instr is MOVK/MOVZ (imm16) with a zero immediate field +inline bool isMoveWideImm16(uint32_t Instr) { + constexpr uint32_t MoveWideImm16Mask = 0x5f9fffe0; + return (Instr & MoveWideImm16Mask) == 0x52800000; +} + +// Returns the amount the address operand of MOVK/MOVZ (imm16) +// should be shifted right by. +// +// The shift value is specfied in the assembly as LSL #. +inline unsigned getMoveWide16Shift(uint32_t Instr) { + if (isMoveWideImm16(Instr)) { + uint32_t ImplicitShift = (Instr >> 21) & 0b11; + return ImplicitShift << 4; + } + + return 0; +} + /// Apply fixup expression for edge to block content. inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { using namespace support; @@ -152,6 +172,20 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { *(ulittle32_t *)FixupPtr = FixedInstr; break; } + case MoveWide16: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue(); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(isMoveWideImm16(RawInstr) && + "RawInstr isn't a MOVK/MOVZ instruction"); + + unsigned ImmShift = getMoveWide16Shift(RawInstr); + uint32_t Imm = (TargetOffset >> ImmShift) & 0xffff; + uint32_t FixedInstr = RawInstr | (Imm << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } case LDRLiteral19: { assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 1397a45df4c2fb..28a6f9ce90d926 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -38,6 +38,8 @@ const char *getEdgeKindName(Edge::Kind R) { return "Page21"; case PageOffset12: return "PageOffset12"; + case MoveWide16: + return "MoveWide16"; case GOTPage21: return "GOTPage21"; case GOTPageOffset12: From 95642af5a565ea5184f8a7a8c22dbb18ed4b68d5 Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Tue, 14 Jun 2022 14:06:08 +0900 Subject: [PATCH 33/34] [JITLink][ELF][AArch64] Implement R_AARCH64_MOVW_UABS_G*_NC. Implements R_AARCH64_MOVW_UABS_G*_NC fixup edges. These relocation entries can be generated when code is compiled without a PIC flag. With this patch, clang-repl can printf Hello World with ObjectLinkerLayer on aarch64 linux. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127585 --- .../ExecutionEngine/JITLink/ELF_aarch64.cpp | 64 +++++++++++++++++++ .../JITLink/AArch64/ELF_aarch64_relocations.s | 54 ++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 9c2dff77658287..4d3e966f7f5032 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -55,6 +55,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { ELFLdSt32Abs12, ELFLdSt64Abs12, ELFLdSt128Abs12, + ELFMovwAbsG0, + ELFMovwAbsG1, + ELFMovwAbsG2, + ELFMovwAbsG3, ELFAbs64, ELFPrel32, ELFPrel64, @@ -83,6 +87,14 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFLdSt64Abs12; case ELF::R_AARCH64_LDST128_ABS_LO12_NC: return ELFLdSt128Abs12; + case ELF::R_AARCH64_MOVW_UABS_G0_NC: + return ELFMovwAbsG0; + case ELF::R_AARCH64_MOVW_UABS_G1_NC: + return ELFMovwAbsG1; + case ELF::R_AARCH64_MOVW_UABS_G2_NC: + return ELFMovwAbsG2; + case ELF::R_AARCH64_MOVW_UABS_G3: + return ELFMovwAbsG3; case ELF::R_AARCH64_ABS64: return ELFAbs64; case ELF::R_AARCH64_PREL32: @@ -216,6 +228,50 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { Kind = aarch64::PageOffset12; break; } + case ELFMovwAbsG0: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 0) + return make_error( + "R_AARCH64_MOVW_UABS_G0_NC target is not a " + "MOVK/MOVZ (imm16, LSL #0) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFMovwAbsG1: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 16) + return make_error( + "R_AARCH64_MOVW_UABS_G1_NC target is not a " + "MOVK/MOVZ (imm16, LSL #16) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFMovwAbsG2: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 32) + return make_error( + "R_AARCH64_MOVW_UABS_G2_NC target is not a " + "MOVK/MOVZ (imm16, LSL #32) instruction"); + + Kind = aarch64::MoveWide16; + break; + } + case ELFMovwAbsG3: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isMoveWideImm16(Instr) || + aarch64::getMoveWide16Shift(Instr) != 48) + return make_error( + "R_AARCH64_MOVW_UABS_G3 target is not a " + "MOVK/MOVZ (imm16, LSL #48) instruction"); + + Kind = aarch64::MoveWide16; + break; + } case ELFAbs64: { Kind = aarch64::Pointer64; break; @@ -268,6 +324,14 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return "ELFLdSt64Abs12"; case ELFLdSt128Abs12: return "ELFLdSt128Abs12"; + case ELFMovwAbsG0: + return "ELFMovwAbsG0"; + case ELFMovwAbsG1: + return "ELFMovwAbsG1"; + case ELFMovwAbsG2: + return "ELFMovwAbsG2"; + case ELFMovwAbsG3: + return "ELFMovwAbsG3"; case ELFAbs64: return "ELFAbs64"; case ELFPrel32: diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s index 8714f09095c830..3eb72e5b2456aa 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s @@ -141,6 +141,60 @@ test_str_64bit: str x0, [x1, :lo12:named_data] .size test_str_64bit, .-test_str_64bit + +# Check R_AARCH64_MOVW_UABS_G*_NC relocation of a local symbol +# +# The immediate value should be the symbol address right shifted according to LSL value +# +# jitlink-check: decode_operand(test_movz_g0_nc, 1) = named_data[15:0] +# jitlink-check: decode_operand(test_movk_g0_nc, 2) = named_data[15:0] +# jitlink-check: decode_operand(test_movz_g1_nc, 1) = named_data[31:16] +# jitlink-check: decode_operand(test_movk_g1_nc, 2) = named_data[31:16] +# jitlink-check: decode_operand(test_movz_g2_nc, 1) = named_data[47:32] +# jitlink-check: decode_operand(test_movk_g2_nc, 2) = named_data[47:32] +# jitlink-check: decode_operand(test_movz_g3, 1) = named_data[63:48] +# jitlink-check: decode_operand(test_movk_g3, 2) = named_data[63:48] + + .globl test_movz_g0_nc +test_movz_g0_nc: + movz x0, #:abs_g0_nc:named_data + .size test_movz_g0_nc, .-test_movz_g0_nc + + .globl test_movk_g0_nc +test_movk_g0_nc: + movk x0, #:abs_g0_nc:named_data + .size test_movk_g0_nc, .-test_movk_g0_nc + + .globl test_movz_g1_nc +test_movz_g1_nc: + movz x0, #:abs_g1_nc:named_data + .size test_movz_g1_nc, .-test_movz_g1_nc + + .globl test_movk_g1_nc +test_movk_g1_nc: + movk x0, #:abs_g1_nc:named_data + .size test_movk_g1_nc, .-test_movk_g1_nc + + .globl test_movz_g2_nc +test_movz_g2_nc: + movz x0, #:abs_g2_nc:named_data + .size test_movz_g2_nc, .-test_movz_g2_nc + + .globl test_movk_g2_nc +test_movk_g2_nc: + movk x0, #:abs_g2_nc:named_data + .size test_movk_g2_nc, .-test_movk_g2_nc + + .globl test_movk_g3 +test_movk_g3: + movk x0, #:abs_g3:named_data + .size test_movk_g3, .-test_movk_g3 + + .globl test_movz_g3 +test_movz_g3: + movz x0, #:abs_g3:named_data + .size test_movz_g3, .-test_movz_g3 + # Check R_AARCH64_ABS64 relocation of a function pointer to local symbol # # jitlink-check: *{8}local_func_addr_quad = named_func From fddc9fd034f0341aa692932d976696fe9daeba5e Mon Sep 17 00:00:00 2001 From: Sunho Kim Date: Wed, 15 Jun 2022 23:24:18 +0900 Subject: [PATCH 34/34] [JITLink][ELF] Log enum name of unsupported relocation type. Logs enum name of unsupported relocation type. This also changes elf/x86 to use common util function (getELFRelocationTypeName) inside llvm object module. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D127715 --- .../lib/ExecutionEngine/JITLink/ELF_aarch64.cpp | 5 +++-- llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp | 5 +++-- llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp | 17 +++-------------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 4d3e966f7f5032..652ad2a0736e47 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -107,8 +107,9 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder { return ELFLd64GOTLo12; } - return make_error("Unsupported aarch64 relocation:" + - formatv("{0:d}", Type)); + return make_error( + "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_AARCH64, Type)); } Error addRelocations() override { diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index f83001417e9462..2534498da136db 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -454,8 +454,9 @@ class ELFLinkGraphBuilder_riscv : public ELFLinkGraphBuilder { return EdgeKind_riscv::R_RISCV_32_PCREL; } - return make_error("Unsupported riscv relocation:" + - formatv("{0:d}", Type)); + return make_error( + "Unsupported riscv relocation:" + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_RISCV, Type)); } Error addRelocations() override { diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index ca9a2964ff146b..8f21274bd1a3c0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -97,17 +97,6 @@ Error buildTables_ELF_x86_64(LinkGraph &G) { } } // namespace -static const char *getELFX86_64RelocName(uint32_t Type) { - switch (Type) { -#define ELF_RELOC(Name, Number) \ - case Number: \ - return #Name; -#include "llvm/BinaryFormat/ELFRelocs/x86_64.def" -#undef ELF_RELOC - } - return "Unrecognized ELF/x86-64 relocation type"; -} - namespace llvm { namespace jitlink { @@ -146,9 +135,9 @@ class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder { case ELF::R_X86_64_TLSGD: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV; } - return make_error("Unsupported x86-64 relocation type " + - formatv("{0:d}: ", Type) + - getELFX86_64RelocName(Type)); + return make_error( + "Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_X86_64, Type)); } Error addRelocations() override {