diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h index 5c77d183e1ef3..bbdbdbbde5322 100644 --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -66,6 +66,14 @@ class BPFIRPeepholePass : public PassInfoMixin { static bool isRequired() { return true; } }; +class BPFASpaceCastSimplifyPass + : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + static bool isRequired() { return true; } +}; + class BPFAdjustOptPass : public PassInfoMixin { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); diff --git a/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp b/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp new file mode 100644 index 0000000000000..f87b299bbba65 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp @@ -0,0 +1,92 @@ +//===-- BPFASpaceCastSimplifyPass.cpp - BPF addrspacecast simplications --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include + +#define DEBUG_TYPE "bpf-aspace-simplify" + +using namespace llvm; + +namespace { + +struct CastGEPCast { + AddrSpaceCastInst *OuterCast; + + // Match chain of instructions: + // %inner = addrspacecast N->M + // %gep = getelementptr %inner, ... + // %outer = addrspacecast M->N %gep + // Where I is %outer. + static std::optional match(Value *I) { + auto *OuterCast = dyn_cast(I); + if (!OuterCast) + return std::nullopt; + auto *GEP = dyn_cast(OuterCast->getPointerOperand()); + if (!GEP) + return std::nullopt; + auto *InnerCast = dyn_cast(GEP->getPointerOperand()); + if (!InnerCast) + return std::nullopt; + if (InnerCast->getSrcAddressSpace() != OuterCast->getDestAddressSpace()) + return std::nullopt; + if (InnerCast->getDestAddressSpace() != OuterCast->getSrcAddressSpace()) + return std::nullopt; + return CastGEPCast{OuterCast}; + } + + static PointerType *changeAddressSpace(PointerType *Ty, unsigned AS) { + return Ty->get(Ty->getContext(), AS); + } + + // Assuming match(this->OuterCast) is true, convert: + // (addrspacecast M->N (getelementptr (addrspacecast N->M ptr) ...)) + // To: + // (getelementptr ptr ...) + GetElementPtrInst *rewrite() { + auto *GEP = cast(OuterCast->getPointerOperand()); + auto *InnerCast = cast(GEP->getPointerOperand()); + unsigned AS = OuterCast->getDestAddressSpace(); + auto *NewGEP = cast(GEP->clone()); + NewGEP->setName(GEP->getName()); + NewGEP->insertAfter(OuterCast); + NewGEP->setOperand(0, InnerCast->getPointerOperand()); + auto *GEPTy = cast(GEP->getType()); + NewGEP->mutateType(changeAddressSpace(GEPTy, AS)); + OuterCast->replaceAllUsesWith(NewGEP); + OuterCast->eraseFromParent(); + if (GEP->use_empty()) + GEP->eraseFromParent(); + if (InnerCast->use_empty()) + InnerCast->eraseFromParent(); + return NewGEP; + } +}; + +} // anonymous namespace + +PreservedAnalyses BPFASpaceCastSimplifyPass::run(Function &F, + FunctionAnalysisManager &AM) { + SmallVector WorkList; + bool Changed = false; + for (BasicBlock &BB : F) { + for (Instruction &I : BB) + if (auto It = CastGEPCast::match(&I)) + WorkList.push_back(It.value()); + Changed |= !WorkList.empty(); + + while (!WorkList.empty()) { + CastGEPCast InsnChain = WorkList.pop_back_val(); + GetElementPtrInst *NewGEP = InsnChain.rewrite(); + for (User *U : NewGEP->users()) + if (auto It = CastGEPCast::match(U)) + WorkList.push_back(It.value()); + } + } + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 08ac4b25540f7..5f26bec2e390c 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -121,6 +121,10 @@ void BPFTargetMachine::registerPassBuilderCallbacks( FPM.addPass(BPFPreserveStaticOffsetPass(false)); return true; } + if (PassName == "bpf-aspace-simplify") { + FPM.addPass(BPFASpaceCastSimplifyPass()); + return true; + } return false; }); PB.registerPipelineStartEPCallback( @@ -135,6 +139,7 @@ void BPFTargetMachine::registerPassBuilderCallbacks( PB.registerPeepholeEPCallback([=](FunctionPassManager &FPM, OptimizationLevel Level) { FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))); + FPM.addPass(BPFASpaceCastSimplifyPass()); }); PB.registerScalarOptimizerLateEPCallback( [=](FunctionPassManager &FPM, OptimizationLevel Level) { diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index d88e7ade40b9a..cb21ed03a86c1 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -24,6 +24,7 @@ add_llvm_target(BPFCodeGen BPFAbstractMemberAccess.cpp BPFAdjustOpt.cpp BPFAsmPrinter.cpp + BPFASpaceCastSimplifyPass.cpp BPFCheckAndAdjustIR.cpp BPFFrameLowering.cpp BPFInstrInfo.cpp diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll new file mode 100644 index 0000000000000..32d67284d1c1b --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass removes unnecessary (for BPF) +; address space casts for cast M->N -> GEP -> cast N->M chain. + +define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(1) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8 +; CHECK-NEXT: ret ptr addrspace(1) [[B1]] +; + entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + %c = addrspacecast ptr %b to ptr addrspace(1) + ret ptr addrspace(1) %c +} diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll new file mode 100644 index 0000000000000..a2965554a9733 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll @@ -0,0 +1,21 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass does not change +; chain 'cast M->N -> GEP -> cast N->K'. + +define dso_local ptr addrspace(2) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(2) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr +; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 +; CHECK-NEXT: [[C:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(2) +; CHECK-NEXT: ret ptr addrspace(2) [[C]] +; + entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + %c = addrspacecast ptr %b to ptr addrspace(2) + ret ptr addrspace(2) %c +} diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll new file mode 100644 index 0000000000000..a7736c462b44b --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll @@ -0,0 +1,26 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that when bpf-aspace-simplify pass modifies chain +; 'cast M->N -> GEP -> cast N->M' it does not remove GEP, +; when that GEP is used by some other instruction. + +define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(1) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr +; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 +; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8 +; CHECK-NEXT: call void @sink(ptr [[B]]) +; CHECK-NEXT: ret ptr addrspace(1) [[B1]] +; + entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + %c = addrspacecast ptr %b to ptr addrspace(1) + call void @sink(ptr %b) + ret ptr addrspace(1) %c +} + +declare dso_local void @sink(ptr) diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll new file mode 100644 index 0000000000000..b2c384bbb6abd --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll @@ -0,0 +1,21 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass simplifies chain +; 'cast K->M -> cast M->N -> GEP -> cast N->M -> cast M->K'. + +define dso_local ptr addrspace(2) @test (ptr addrspace(2) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(2) @test( +; CHECK-SAME: ptr addrspace(2) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C12:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[P]], i64 8 +; CHECK-NEXT: ret ptr addrspace(2) [[C12]] +; + entry: + %a = addrspacecast ptr addrspace(2) %p to ptr addrspace(1) + %b = addrspacecast ptr addrspace(1) %a to ptr + %c = getelementptr inbounds i8, ptr %b, i64 8 + %d = addrspacecast ptr %c to ptr addrspace(1) + %e = addrspacecast ptr addrspace (1) %d to ptr addrspace(2) + ret ptr addrspace(2) %e +} diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll new file mode 100644 index 0000000000000..b62d25384d958 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll @@ -0,0 +1,25 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s + +; Check that bpf-aspace-simplify pass removes unnecessary (for BPF) +; address space casts for cast M->N -> GEP -> cast N->M chain, +; where chain is split between several BBs. + +define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) { +; CHECK-LABEL: define dso_local ptr addrspace(1) @test( +; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8 +; CHECK-NEXT: ret ptr addrspace(1) [[B1]] +; +entry: + %a = addrspacecast ptr addrspace(1) %p to ptr + %b = getelementptr inbounds i8, ptr %a, i64 8 + br label %exit + +exit: + %c = addrspacecast ptr %b to ptr addrspace(1) + ret ptr addrspace(1) %c +}