[PAC][CodeGen][ELF][AArch64] Support signed GOT

This re-applies llvm#96164 after revert in llvm#102434.

Support the following relocations and assembly operators:

- `R_AARCH64_AUTH_ADR_GOT_PAGE` (`:got_auth:` for `adrp`)
- `R_AARCH64_AUTH_LD64_GOT_LO12_NC` (`:got_auth_lo12:` for `ldr`)
- `R_AARCH64_AUTH_GOT_ADD_LO12_NC` (`:got_auth_lo12:` for `add`)

`LOADgotAUTH` pseudo-instruction is introduced which is later expanded to actual instruction sequence like the following.

```
adrp x16, :got_auth:sym
add x16, x16, :got_auth_lo12:sym
ldr x0, [x16]
autia x0, x16
```

If a resign is requested, like below, `LOADgotPAC` pseudo is used, and GOT load is lowered similarly to `LOADgotAUTH`.

```
@var = global i32 0
define ptr @resign_globalvar() {
  ret ptr ptrauth (ptr @var, i32 3, i64 43)
}
```

If FPAC bit is not set and resign is requested, a check+trap sequence similar to one used for `AUT` pseudo is emitted.

Both SelectionDAG and GlobalISel are suppported.
For FastISel, we fall back to SelectionDAG.

Tests starting with 'ptrauth-' have corresponding variants w/o this prefix.

See also specification
https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#appendix-signed-got

Author: kovdan01

llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

@@ -160,6 +160,11 @@ class AArch64AsmPrinter : public AsmPrinter {
   // adrp-add followed by PAC sign)
   void LowerMOVaddrPAC(const MachineInstr &MI);
 
+  // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
+  // and authenticate it with, if FPAC bit is not set, check+trap sequence after
+  // authenticating)
+  void LowerLOADgotAUTH(const MachineInstr &MI);
+
   /// tblgen'erated driver function for lowering simple MI->MC
   /// pseudo instructions.
   bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
@@ -923,6 +928,22 @@ void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
 
       OutStreamer->addBlankLine();
     }
+
+    // With signed ELF GOT enabled, the linker looks at the symbol type to
+    // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
+    // for functions not defined in the module have STT_NOTYPE type by default.
+    // This makes linker to emit signing schema with DA key (instead of IA) for
+    // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
+    // all function symbols used in the module to have STT_FUNC type. See
+    // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
+    const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
+        M.getModuleFlag("ptrauth-elf-got"));
+    if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
+      for (const GlobalValue &GV : M.global_values())
+        if (!GV.use_empty() && GV.getValueType()->isFunctionTy() &&
+            !GV.getName().starts_with("llvm."))
+          OutStreamer->emitSymbolAttribute(getSymbol(&GV),
+                                           MCSA_ELF_TypeFunction);
   }
 
   // Emit stack and fault map information.
@@ -2168,6 +2189,10 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
   };
 
   const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
+  const bool IsELFSignedGOT = MI.getParent()
+                                  ->getParent()
+                                  ->getInfo<AArch64FunctionInfo>()
+                                  ->hasELFSignedGOT();
   MachineOperand GAOp = MI.getOperand(0);
   const uint64_t KeyC = MI.getOperand(1).getImm();
   assert(KeyC <= AArch64PACKey::LAST &&
@@ -2184,9 +2209,17 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
   // Emit:
   // target materialization:
   // - via GOT:
-  //     adrp x16, :got:target
-  //     ldr x16, [x16, :got_lo12:target]
-  //     add offset to x16 if offset != 0
+  //   - unsigned GOT:
+  //       adrp x16, :got:target
+  //       ldr x16, [x16, :got_lo12:target]
+  //       add offset to x16 if offset != 0
+  //   - ELF signed GOT:
+  //       adrp x17, :got:target
+  //       add x17, x17, :got_auth_lo12:target
+  //       ldr x16, [x17]
+  //       aut{i|d}a x16, x17
+  //       check+trap sequence (if no FPAC)
+  //       add offset to x16 if offset != 0
   //
   // - direct:
   //     adrp x16, target
@@ -2229,13 +2262,79 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
   MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
 
   EmitAndIncrement(
-      MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi));
+      MCInstBuilder(AArch64::ADRP)
+          .addReg(IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
+          .addOperand(GAMCHi));
 
   if (IsGOTLoad) {
-    EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
-                         .addReg(AArch64::X16)
-                         .addReg(AArch64::X16)
-                         .addOperand(GAMCLo));
+    if (IsELFSignedGOT) {
+      EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
+                           .addReg(AArch64::X17)
+                           .addReg(AArch64::X17)
+                           .addOperand(GAMCLo)
+                           .addImm(0));
+
+      EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
+                           .addReg(AArch64::X16)
+                           .addReg(AArch64::X17)
+                           .addImm(0));
+
+      assert(GAOp.isGlobal());
+      assert(GAOp.getGlobal()->getValueType() != nullptr);
+      unsigned AuthOpcode = GAOp.getGlobal()->getValueType()->isFunctionTy()
+                                ? AArch64::AUTIA
+                                : AArch64::AUTDA;
+
+      EmitAndIncrement(MCInstBuilder(AuthOpcode)
+                           .addReg(AArch64::X16)
+                           .addReg(AArch64::X16)
+                           .addReg(AArch64::X17));
+
+      if (!STI->hasFPAC()) {
+        auto AuthKey = (AuthOpcode == AArch64::AUTIA ? AArch64PACKey::IA
+                                                     : AArch64PACKey::DA);
+        unsigned XPACOpc = getXPACOpcodeForKey(AuthKey);
+        MCSymbol *SuccessSym = createTempSymbol("auth_success_");
+
+        // XPAC has tied src/dst: use x17 as a temporary copy.
+        //  mov x17, x16
+        EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs)
+                             .addReg(AArch64::X17)
+                             .addReg(AArch64::XZR)
+                             .addReg(AArch64::X16)
+                             .addImm(0));
+
+        //  xpaci x17
+        EmitAndIncrement(
+            MCInstBuilder(XPACOpc).addReg(AArch64::X17).addReg(AArch64::X17));
+
+        //  cmp x16, x17
+        EmitAndIncrement(MCInstBuilder(AArch64::SUBSXrs)
+                             .addReg(AArch64::XZR)
+                             .addReg(AArch64::X16)
+                             .addReg(AArch64::X17)
+                             .addImm(0));
+
+        //  b.eq Lsuccess
+        EmitAndIncrement(
+            MCInstBuilder(AArch64::Bcc)
+                .addImm(AArch64CC::EQ)
+                .addExpr(MCSymbolRefExpr::create(SuccessSym, OutContext)));
+
+        // Trapping sequences do a 'brk'.
+        //  brk #<0xc470 + aut key>
+        EmitAndIncrement(MCInstBuilder(AArch64::BRK).addImm(0xc470 | AuthKey));
+
+        // If the auth check succeeds, we can continue.
+        // Lsuccess:
+        OutStreamer->emitLabel(SuccessSym);
+      }
+    } else {
+      EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
+                           .addReg(AArch64::X16)
+                           .addReg(AArch64::X16)
+                           .addOperand(GAMCLo));
+    }
   } else {
     EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
                          .addReg(AArch64::X16)
@@ -2320,6 +2419,53 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
   assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
 }
 
+void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
+  unsigned InstsEmitted = 0;
+  auto EmitAndIncrement = [this, &InstsEmitted](const MCInst &Inst) {
+    EmitToStreamer(*OutStreamer, Inst);
+    ++InstsEmitted;
+  };
+
+  Register DstReg = MI.getOperand(0).getReg();
+  const MachineOperand &GAMO = MI.getOperand(1);
+  assert(GAMO.getOffset() == 0);
+
+  MachineOperand GAHiOp(GAMO);
+  MachineOperand GALoOp(GAMO);
+  GAHiOp.addTargetFlag(AArch64II::MO_PAGE);
+  GALoOp.addTargetFlag(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
+
+  MCOperand GAMCHi, GAMCLo;
+  MCInstLowering.lowerOperand(GAHiOp, GAMCHi);
+  MCInstLowering.lowerOperand(GALoOp, GAMCLo);
+
+  EmitAndIncrement(
+      MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi));
+
+  EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
+                       .addReg(AArch64::X16)
+                       .addReg(AArch64::X16)
+                       .addOperand(GAMCLo)
+                       .addImm(0));
+
+  EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
+                       .addReg(DstReg)
+                       .addReg(AArch64::X16)
+                       .addImm(0));
+
+  assert(GAMO.isGlobal());
+  assert(GAMO.getGlobal()->getValueType() != nullptr);
+  unsigned AuthOpcode = GAMO.getGlobal()->getValueType()->isFunctionTy()
+                            ? AArch64::AUTIA
+                            : AArch64::AUTDA;
+  EmitAndIncrement(MCInstBuilder(AuthOpcode)
+                       .addReg(DstReg)
+                       .addReg(DstReg)
+                       .addReg(AArch64::X16));
+
+  assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
+}
+
 const MCExpr *
 AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
   const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
@@ -2484,6 +2630,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
     LowerMOVaddrPAC(*MI);
     return;
 
+  case AArch64::LOADgotAUTH:
+    LowerLOADgotAUTH(*MI);
+    return;
+
   case AArch64::BRA:
   case AArch64::BLRA:
     emitPtrauthBranch(MI);

llvm/lib/Target/AArch64/AArch64FastISel.cpp

@@ -453,6 +453,9 @@ unsigned AArch64FastISel::materializeGV(const GlobalValue *GV) {
   if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO())
     return 0;
 
+  if (FuncInfo.MF->getInfo<AArch64FunctionInfo>()->hasELFSignedGOT())
+    return 0;
+
   unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, TM);
 
   EVT DestEVT = TLI.getValueType(DL, GV->getType(), true);

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -9396,6 +9396,11 @@ SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
   SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
   // FIXME: Once remat is capable of dealing with instructions with register
   // operands, expand this into two nodes instead of using a wrapper node.
+  if (DAG.getMachineFunction()
+          .getInfo<AArch64FunctionInfo>()
+          ->hasELFSignedGOT())
+    return SDValue(DAG.getMachineNode(AArch64::LOADgotAUTH, DL, Ty, GotAddr),
+                   0);
   return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
 }
 

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -1887,8 +1887,9 @@ let Predicates = [HasPAuth] in {
                Sched<[WriteI, ReadI]> {
     let isReMaterializable = 1;
     let isCodeGenOnly = 1;
-    let Size = 40; // 12 fixed + 28 variable, for pointer offset, and discriminator
-    let Defs = [X16,X17];
+    let Size = 68; // 12 fixed + 56 variable, for pointer offset, discriminator and
+                   // ELF signed GOT signed pointer authentication (if no FPAC)
+    let Defs = [X16,X17,NZCV];
   }
 
   // Load a signed global address from a special $auth_ptr$ stub slot.
@@ -1926,6 +1927,12 @@ let Predicates = [HasPAuth] in {
                                 tcGPR64:$AddrDisc),
               (AUTH_TCRETURN_BTI tcGPRx16x17:$dst, imm:$FPDiff, imm:$Key,
                                  imm:$Disc, tcGPR64:$AddrDisc)>;
+
+  def LOADgotAUTH : Pseudo<(outs GPR64common:$dst), (ins i64imm:$addr), []>,
+               Sched<[WriteI, ReadI]> {
+    let Defs = [X16];
+    let Size = 16;
+  }
 }
 
 // v9.5-A pointer authentication extensions

llvm/lib/Target/AArch64/AArch64MCInstLower.cpp

@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64MCInstLower.h"
+#include "AArch64MachineFunctionInfo.h"
 #include "MCTargetDesc/AArch64MCExpr.h"
 #include "Utils/AArch64BaseInfo.h"
 #include "llvm/CodeGen/AsmPrinter.h"
@@ -185,9 +186,12 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
                                                     MCSymbol *Sym) const {
   uint32_t RefFlags = 0;
 
-  if (MO.getTargetFlags() & AArch64II::MO_GOT)
-    RefFlags |= AArch64MCExpr::VK_GOT;
-  else if (MO.getTargetFlags() & AArch64II::MO_TLS) {
+  if (MO.getTargetFlags() & AArch64II::MO_GOT) {
+    const MachineFunction *MF = MO.getParent()->getParent()->getParent();
+    RefFlags |= (MF->getInfo<AArch64FunctionInfo>()->hasELFSignedGOT()
+                     ? AArch64MCExpr::VK_GOT_AUTH
+                     : AArch64MCExpr::VK_GOT);
+  } else if (MO.getTargetFlags() & AArch64II::MO_TLS) {
     TLSModel::Model Model;
     if (MO.isGlobal()) {
       const GlobalValue *GV = MO.getGlobal();

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp

@@ -72,6 +72,18 @@ static bool ShouldSignWithBKey(const Function &F, const AArch64Subtarget &STI) {
   return Key == "b_key";
 }
 
+static bool HasELFSignedGOTHelper(const Function &F,
+                                  const AArch64Subtarget *STI) {
+  if (!Triple(STI->getTargetTriple()).isOSBinFormatELF())
+    return false;
+  const Module *M = F.getParent();
+  const auto *Flag = mdconst::extract_or_null<ConstantInt>(
+      M->getModuleFlag("ptrauth-elf-got"));
+  if (Flag && Flag->getZExtValue() == 1)
+    return true;
+  return false;
+}
+
 AArch64FunctionInfo::AArch64FunctionInfo(const Function &F,
                                          const AArch64Subtarget *STI) {
   // If we already know that the function doesn't have a redzone, set
@@ -80,6 +92,7 @@ AArch64FunctionInfo::AArch64FunctionInfo(const Function &F,
     HasRedZone = false;
   std::tie(SignReturnAddress, SignReturnAddressAll) = GetSignReturnAddress(F);
   SignWithBKey = ShouldSignWithBKey(F, *STI);
+  HasELFSignedGOT = HasELFSignedGOTHelper(F, STI);
   // TODO: skip functions that have no instrumented allocas for optimization
   IsMTETagged = F.hasFnAttribute(Attribute::SanitizeMemTag);
 

llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h

@@ -177,6 +177,11 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
   /// SignWithBKey modifies the default PAC-RET mode to signing with the B key.
   bool SignWithBKey = false;
 
+  /// HasELFSignedGOT is true if the target binary format is ELF and the IR
+  /// module containing the corresponding function has "ptrauth-elf-got" flag
+  /// set to 1.
+  bool HasELFSignedGOT = false;
+
   /// SigningInstrOffset captures the offset of the PAC-RET signing instruction
   /// within the prologue, so it can be re-used for authentication in the
   /// epilogue when using PC as a second salt (FEAT_PAuth_LR)
@@ -509,6 +514,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
 
   bool shouldSignWithBKey() const { return SignWithBKey; }
 
+  bool hasELFSignedGOT() const { return HasELFSignedGOT; }
+
   MCSymbol *getSigningInstrLabel() const { return SignInstrLabel; }
   void setSigningInstrLabel(MCSymbol *Label) { SignInstrLabel = Label; }