diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index 076719abd0356..94a2d0c6477b2 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -24,6 +24,7 @@ #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CallBrPrepare.h" #include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/CodeGen/ReplaceWithVeclib.h" diff --git a/llvm/include/llvm/CodeGen/InterleavedAccess.h b/llvm/include/llvm/CodeGen/InterleavedAccess.h new file mode 100644 index 0000000000000..31bd19a3191a2 --- /dev/null +++ b/llvm/include/llvm/CodeGen/InterleavedAccess.h @@ -0,0 +1,34 @@ +//===---- llvm/CodeGen/InterleavedAccess.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declaration of the InterleavedAccessPass class, +/// its corresponding pass name is `interleaved-access`. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_INTERLEAVEDACCESS_H +#define LLVM_CODEGEN_INTERLEAVEDACCESS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class TargetMachine; + +class InterleavedAccessPass : public PassInfoMixin { + const TargetMachine *TM; + +public: + explicit InterleavedAccessPass(const TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_INTERLEAVEDACCESS_H diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index 1e9e5838841b2..ceca04cd5ba3a 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -43,6 +43,7 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass, ()) FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ()) FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ()) FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ()) +FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM)) FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ()) FUNCTION_PASS("lowerinvoke", LowerInvokePass, ()) FUNCTION_PASS("mergeicmps", MergeICmpsPass, ()) @@ -127,7 +128,6 @@ DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ()) DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ()) -DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ()) DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ()) DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) DUMMY_FUNCTION_PASS("sjljehprepare", SjLjEHPreparePass, ()) diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp index 65a6859a006a5..2a0daf404c978 100644 --- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp +++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp @@ -48,6 +48,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -82,22 +83,14 @@ static cl::opt LowerInterleavedAccesses( namespace { -class InterleavedAccess : public FunctionPass { -public: - static char ID; - - InterleavedAccess() : FunctionPass(ID) { - initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); - } +class InterleavedAccessImpl { + friend class InterleavedAccess; - StringRef getPassName() const override { return "Interleaved Access Pass"; } - - bool runOnFunction(Function &F) override; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.setPreservesCFG(); - } +public: + InterleavedAccessImpl() = default; + InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI) + : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {} + bool runOnFunction(Function &F); private: DominatorTree *DT = nullptr; @@ -141,10 +134,60 @@ class InterleavedAccess : public FunctionPass { LoadInst *LI); }; +class InterleavedAccess : public FunctionPass { + InterleavedAccessImpl Impl; + +public: + static char ID; + + InterleavedAccess() : FunctionPass(ID) { + initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { return "Interleaved Access Pass"; } + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.setPreservesCFG(); + } +}; + } // end anonymous namespace. +PreservedAnalyses InterleavedAccessPass::run(Function &F, + FunctionAnalysisManager &FAM) { + auto *DT = &FAM.getResult(F); + auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); + InterleavedAccessImpl Impl(DT, TLI); + bool Changed = Impl.runOnFunction(F); + + if (!Changed) + return PreservedAnalyses::all(); + + PreservedAnalyses PA; + PA.preserveSet(); + return PA; +} + char InterleavedAccess::ID = 0; +bool InterleavedAccess::runOnFunction(Function &F) { + auto *TPC = getAnalysisIfAvailable(); + if (!TPC || !LowerInterleavedAccesses) + return false; + + LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); + + Impl.DT = &getAnalysis().getDomTree(); + auto &TM = TPC->getTM(); + Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering(); + Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor(); + + return Impl.runOnFunction(F); +} + INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE, "Lower interleaved memory accesses to target specific intrinsics", false, false) @@ -228,7 +271,7 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor, return false; } -bool InterleavedAccess::lowerInterleavedLoad( +bool InterleavedAccessImpl::lowerInterleavedLoad( LoadInst *LI, SmallVector &DeadInsts) { if (!LI->isSimple() || isa(LI->getType())) return false; @@ -334,7 +377,7 @@ bool InterleavedAccess::lowerInterleavedLoad( return true; } -bool InterleavedAccess::replaceBinOpShuffles( +bool InterleavedAccessImpl::replaceBinOpShuffles( ArrayRef BinOpShuffles, SmallVectorImpl &Shuffles, LoadInst *LI) { for (auto *SVI : BinOpShuffles) { @@ -367,7 +410,7 @@ bool InterleavedAccess::replaceBinOpShuffles( return !BinOpShuffles.empty(); } -bool InterleavedAccess::tryReplaceExtracts( +bool InterleavedAccessImpl::tryReplaceExtracts( ArrayRef Extracts, ArrayRef Shuffles) { // If there aren't any extractelement instructions to modify, there's nothing @@ -431,7 +474,7 @@ bool InterleavedAccess::tryReplaceExtracts( return true; } -bool InterleavedAccess::lowerInterleavedStore( +bool InterleavedAccessImpl::lowerInterleavedStore( StoreInst *SI, SmallVector &DeadInsts) { if (!SI->isSimple()) return false; @@ -457,7 +500,7 @@ bool InterleavedAccess::lowerInterleavedStore( return true; } -bool InterleavedAccess::lowerDeinterleaveIntrinsic( +bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic( IntrinsicInst *DI, SmallVector &DeadInsts) { LoadInst *LI = dyn_cast(DI->getOperand(0)); @@ -476,7 +519,7 @@ bool InterleavedAccess::lowerDeinterleaveIntrinsic( return true; } -bool InterleavedAccess::lowerInterleaveIntrinsic( +bool InterleavedAccessImpl::lowerInterleaveIntrinsic( IntrinsicInst *II, SmallVector &DeadInsts) { if (!II->hasOneUse()) return false; @@ -498,18 +541,7 @@ bool InterleavedAccess::lowerInterleaveIntrinsic( return true; } -bool InterleavedAccess::runOnFunction(Function &F) { - auto *TPC = getAnalysisIfAvailable(); - if (!TPC || !LowerInterleavedAccesses) - return false; - - LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); - - DT = &getAnalysis().getDomTree(); - auto &TM = TPC->getTM(); - TLI = TM.getSubtargetImpl(F)->getTargetLowering(); - MaxFactor = TLI->getMaxSupportedInterleaveFactor(); - +bool InterleavedAccessImpl::runOnFunction(Function &F) { // Holds dead instructions that will be erased later. SmallVector DeadInsts; bool Changed = false; diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index a5f9b5424358e..f26450e941870 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -77,6 +77,7 @@ #include "llvm/CodeGen/ExpandLargeDivRem.h" #include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/HardwareLoops.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/TypePromotion.h" #include "llvm/CodeGen/WasmEHPrepare.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 7462704ec2df8..c1641ea8b5b19 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -318,6 +318,7 @@ FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings()) FUNCTION_PASS("instcount", InstCountPass()) FUNCTION_PASS("instnamer", InstructionNamerPass()) FUNCTION_PASS("instsimplify", InstSimplifyPass()) +FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM)) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("irce", IRCEPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll index dd08172be1b87..22df002dd62cd 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll index 2e8a7cf42ac50..399fa5298b7cc 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll index ab70d623470c8..224a0693bf218 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=NEON ; RUN: opt < %s -interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=NEON +; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll index db031ed12b7ec..e48dc5d3051bf 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll index af472630b9518..ea33590cb241b 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll index 77fbb8e7f2cad..14986d9eb85c5 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll @@ -1,5 +1,7 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON ; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON +; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll index 77fbb8e7f2cad..14986d9eb85c5 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll @@ -1,5 +1,7 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON ; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON +; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll index c04464b2ca9df..6353bf10d57c4 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll index 94f63e5921f8b..feb22aa1a3763 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll index 3ea00de0dcb15..77b376b3580ca 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll index 0c56f71cc3693..8934941678723 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll index 9ea1c5e94a983..aed843723189b 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll @@ -2,6 +2,9 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON ; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON +; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll index c84d759ae0fb3..8123ea5bbe9f1 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll @@ -2,6 +2,9 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON ; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON +; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll index 736a7ce4ecf0c..ea2d1ca722ca0 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv32 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV32 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV64 +; RUN: opt < %s -mtriple=riscv32 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV32 +; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV64 define void @load_factor2(ptr %ptr) { ; RV32-LABEL: @load_factor2( diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll index e7b7ec052f459..ac3cab6638b8c 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE32X ; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE64X +; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE32X +; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE64X define <4 x i1> @load_large_vector(ptr %p) { ; ZVE32X-LABEL: @load_large_vector( diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll index 9c896796760ff..a94e6a70e79e6 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -interleaved-access -S | FileCheck %s -check-prefix=ZVL32B ; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVL128B +; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL32B +; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL128B ; Make sure that we don't lower interleaved loads that won't fit into the minimum vlen diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll index a4b4e3748b463..167d282edb3e7 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -interleaved-access -S %s | FileCheck %s +; RUN: opt -passes=interleaved-access -S %s | FileCheck %s target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.15.0" diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll index b4128dda7cee7..13076c7ffec9c 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s ; This file tests the function `llvm::lowerInterleavedLoad/Store`. diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll index ec905a999dca1..6972afe486bd0 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s ; This file tests the function `llvm::lowerInterleavedLoad/Store`. diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll index 9b38f2dfbabd8..327a1d2c6a66c 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -passes=interleaved-access -S | FileCheck %s define <32 x i8> @interleaved_load_vf32_i8_stride3(ptr %ptr){ ; CHECK-LABEL: @interleaved_load_vf32_i8_stride3( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll index 86c1862762581..68d8b94755178 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -passes=interleaved-access -S | FileCheck %s define <32 x i8> @interleaved_load_vf32_i8_stride3(ptr %ptr){ ; CHECK-LABEL: @interleaved_load_vf32_i8_stride3( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll index 301abda5708cf..d1eea7a0d1bfd 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, ptr %p) { ; CHECK-LABEL: @interleaved_store_vf32_i8_stride4( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll index 75595463c7404..3e7bc4130a8cd 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, ptr %p) { ; CHECK-LABEL: @interleaved_store_vf32_i8_stride4(