Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CodeGen] Port InterleavedAccess to new pass manager #74904

Merged
merged 1 commit into from
Dec 10, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
#include "llvm/CodeGen/CallBrPrepare.h"
#include "llvm/CodeGen/ExpandReductions.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/CodeGen/ReplaceWithVeclib.h"
34 changes: 34 additions & 0 deletions llvm/include/llvm/CodeGen/InterleavedAccess.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//===---- llvm/CodeGen/InterleavedAccess.h ----------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file contains the declaration of the InterleavedAccessPass class,
/// its corresponding pass name is `interleaved-access`.
///
//===----------------------------------------------------------------------===//

#ifndef LLVM_CODEGEN_INTERLEAVEDACCESS_H
#define LLVM_CODEGEN_INTERLEAVEDACCESS_H

#include "llvm/IR/PassManager.h"

namespace llvm {

class TargetMachine;

class InterleavedAccessPass : public PassInfoMixin<InterleavedAccessPass> {
const TargetMachine *TM;

public:
explicit InterleavedAccessPass(const TargetMachine *TM) : TM(TM) {}
PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
};

} // namespace llvm

#endif // LLVM_CODEGEN_INTERLEAVEDACCESS_H
2 changes: 1 addition & 1 deletion llvm/include/llvm/CodeGen/MachinePassRegistry.def
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass, ())
FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ())
FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ())
FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ())
FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM))
FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ())
FUNCTION_PASS("lowerinvoke", LowerInvokePass, ())
FUNCTION_PASS("mergeicmps", MergeICmpsPass, ())
@@ -127,7 +128,6 @@ DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ())
DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ())
DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ())
DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ())
DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ())
DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
DUMMY_FUNCTION_PASS("sjljehprepare", SjLjEHPreparePass, ())
98 changes: 65 additions & 33 deletions llvm/lib/CodeGen/InterleavedAccessPass.cpp
Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -82,22 +83,14 @@ static cl::opt<bool> LowerInterleavedAccesses(

namespace {

class InterleavedAccess : public FunctionPass {
public:
static char ID;

InterleavedAccess() : FunctionPass(ID) {
initializeInterleavedAccessPass(*PassRegistry::getPassRegistry());
}
class InterleavedAccessImpl {
friend class InterleavedAccess;

StringRef getPassName() const override { return "Interleaved Access Pass"; }

bool runOnFunction(Function &F) override;

void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<DominatorTreeWrapperPass>();
AU.setPreservesCFG();
}
public:
InterleavedAccessImpl() = default;
InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI)
: DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {}
bool runOnFunction(Function &F);

private:
DominatorTree *DT = nullptr;
@@ -141,10 +134,60 @@ class InterleavedAccess : public FunctionPass {
LoadInst *LI);
};

class InterleavedAccess : public FunctionPass {
InterleavedAccessImpl Impl;

public:
static char ID;

InterleavedAccess() : FunctionPass(ID) {
initializeInterleavedAccessPass(*PassRegistry::getPassRegistry());
}

StringRef getPassName() const override { return "Interleaved Access Pass"; }

bool runOnFunction(Function &F) override;

void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<DominatorTreeWrapperPass>();
AU.setPreservesCFG();
}
};

} // end anonymous namespace.

PreservedAnalyses InterleavedAccessPass::run(Function &F,
FunctionAnalysisManager &FAM) {
auto *DT = &FAM.getResult<DominatorTreeAnalysis>(F);
auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering();
InterleavedAccessImpl Impl(DT, TLI);
bool Changed = Impl.runOnFunction(F);

if (!Changed)
return PreservedAnalyses::all();

PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
return PA;
}

char InterleavedAccess::ID = 0;

bool InterleavedAccess::runOnFunction(Function &F) {
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
if (!TPC || !LowerInterleavedAccesses)
return false;

LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This kind of printing I think is redundant with regular pass debugging statements

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just copy the old code here.


Impl.DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
auto &TM = TPC->getTM<TargetMachine>();
Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering();
Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor();

return Impl.runOnFunction(F);
}

INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE,
"Lower interleaved memory accesses to target specific intrinsics", false,
false)
@@ -228,7 +271,7 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor,
return false;
}

bool InterleavedAccess::lowerInterleavedLoad(
bool InterleavedAccessImpl::lowerInterleavedLoad(
LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))
return false;
@@ -334,7 +377,7 @@ bool InterleavedAccess::lowerInterleavedLoad(
return true;
}

bool InterleavedAccess::replaceBinOpShuffles(
bool InterleavedAccessImpl::replaceBinOpShuffles(
ArrayRef<ShuffleVectorInst *> BinOpShuffles,
SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) {
for (auto *SVI : BinOpShuffles) {
@@ -367,7 +410,7 @@ bool InterleavedAccess::replaceBinOpShuffles(
return !BinOpShuffles.empty();
}

bool InterleavedAccess::tryReplaceExtracts(
bool InterleavedAccessImpl::tryReplaceExtracts(
ArrayRef<ExtractElementInst *> Extracts,
ArrayRef<ShuffleVectorInst *> Shuffles) {
// If there aren't any extractelement instructions to modify, there's nothing
@@ -431,7 +474,7 @@ bool InterleavedAccess::tryReplaceExtracts(
return true;
}

bool InterleavedAccess::lowerInterleavedStore(
bool InterleavedAccessImpl::lowerInterleavedStore(
StoreInst *SI, SmallVector<Instruction *, 32> &DeadInsts) {
if (!SI->isSimple())
return false;
@@ -457,7 +500,7 @@ bool InterleavedAccess::lowerInterleavedStore(
return true;
}

bool InterleavedAccess::lowerDeinterleaveIntrinsic(
bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
IntrinsicInst *DI, SmallVector<Instruction *, 32> &DeadInsts) {
LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0));

@@ -476,7 +519,7 @@ bool InterleavedAccess::lowerDeinterleaveIntrinsic(
return true;
}

bool InterleavedAccess::lowerInterleaveIntrinsic(
bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
IntrinsicInst *II, SmallVector<Instruction *, 32> &DeadInsts) {
if (!II->hasOneUse())
return false;
@@ -498,18 +541,7 @@ bool InterleavedAccess::lowerInterleaveIntrinsic(
return true;
}

bool InterleavedAccess::runOnFunction(Function &F) {
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
if (!TPC || !LowerInterleavedAccesses)
return false;

LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");

DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
auto &TM = TPC->getTM<TargetMachine>();
TLI = TM.getSubtargetImpl(F)->getTargetLowering();
MaxFactor = TLI->getMaxSupportedInterleaveFactor();

bool InterleavedAccessImpl::runOnFunction(Function &F) {
// Holds dead instructions that will be erased later.
SmallVector<Instruction *, 32> DeadInsts;
bool Changed = false;
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassBuilder.cpp
Original file line number Diff line number Diff line change
@@ -77,6 +77,7 @@
#include "llvm/CodeGen/ExpandLargeDivRem.h"
#include "llvm/CodeGen/ExpandLargeFpConvert.h"
#include "llvm/CodeGen/HardwareLoops.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/SafeStack.h"
#include "llvm/CodeGen/TypePromotion.h"
#include "llvm/CodeGen/WasmEHPrepare.h"
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassRegistry.def
Original file line number Diff line number Diff line change
@@ -318,6 +318,7 @@ FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings())
FUNCTION_PASS("instcount", InstCountPass())
FUNCTION_PASS("instnamer", InstructionNamerPass())
FUNCTION_PASS("instsimplify", InstSimplifyPass())
FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM))
FUNCTION_PASS("invalidate<all>", InvalidateAllAnalysesPass())
FUNCTION_PASS("irce", IRCEPass())
FUNCTION_PASS("jump-threading", JumpThreadingPass())
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=NEON
; RUN: opt < %s -interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=NEON
; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED

target triple = "aarch64-linux-gnu"

Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON
; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target triple = "aarch64-linux-gnu"

Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -interleaved-access -S | FileCheck %s
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s

target triple = "aarch64-linux-gnu"

Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Original file line number Diff line number Diff line change
@@ -2,6 +2,9 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Original file line number Diff line number Diff line change
@@ -2,6 +2,9 @@
; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE
; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON
; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE
; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE

target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
target triple = "arm---eabi"
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=riscv32 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV32
; RUN: opt < %s -mtriple=riscv64 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV64
; RUN: opt < %s -mtriple=riscv32 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV32
; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV64

define void @load_factor2(ptr %ptr) {
; RV32-LABEL: @load_factor2(
2 changes: 2 additions & 0 deletions llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE32X
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE64X
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE32X
; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE64X

define <4 x i1> @load_large_vector(ptr %p) {
; ZVE32X-LABEL: @load_large_vector(
2 changes: 2 additions & 0 deletions llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -interleaved-access -S | FileCheck %s -check-prefix=ZVL32B
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVL128B
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL32B
; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL128B

; Make sure that we don't lower interleaved loads that won't fit into the minimum vlen

Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -interleaved-access -S %s | FileCheck %s
; RUN: opt -passes=interleaved-access -S %s | FileCheck %s

target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.15.0"
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s

; This file tests the function `llvm::lowerInterleavedLoad/Store`.

Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s
; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s

; This file tests the function `llvm::lowerInterleavedLoad/Store`.

Loading