Skip to content

Commit

Permalink
[VectorCombine] Do not try to operate on OperandBundles.
Browse files Browse the repository at this point in the history 
This bails out if we see an intrinsic with an operand bundle on it, to make
sure we don't process the bundles incorrectly.

Fixes #110382.
  • Loading branch information
@davemgreen
davemgreen committed Oct 9, 2024
1 parent 587f31f commit c3ca15a
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 28 deletions.
59 changes: 31 additions & 28 deletions llvm/lib/Transforms/Vectorize/VectorCombine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1984,33 +1984,35 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {

// We need each element to be the same type of value, and check that each
// element has a single use.
if (all_of(drop_begin(Item), [Item](InstLane IL) {
Value *FrontV = Item.front().first->get();
if (!IL.first)
return true;
Value *V = IL.first->get();
if (auto *I = dyn_cast<Instruction>(V); I && !I->hasOneUse())
return false;
if (V->getValueID() != FrontV->getValueID())
return false;
if (auto *CI = dyn_cast<CmpInst>(V))
if (CI->getPredicate() != cast<CmpInst>(FrontV)->getPredicate())
return false;
if (auto *CI = dyn_cast<CastInst>(V))
if (CI->getSrcTy() != cast<CastInst>(FrontV)->getSrcTy())
return false;
if (auto *SI = dyn_cast<SelectInst>(V))
if (!isa<VectorType>(SI->getOperand(0)->getType()) ||
SI->getOperand(0)->getType() !=
cast<SelectInst>(FrontV)->getOperand(0)->getType())
return false;
if (isa<CallInst>(V) && !isa<IntrinsicInst>(V))
return false;
auto *II = dyn_cast<IntrinsicInst>(V);
return !II || (isa<IntrinsicInst>(FrontV) &&
II->getIntrinsicID() ==
cast<IntrinsicInst>(FrontV)->getIntrinsicID());
})) {
auto CheckLaneIsEquivalentToFirst = [Item](InstLane IL) {
Value *FrontV = Item.front().first->get();
if (!IL.first)
return true;
Value *V = IL.first->get();
if (auto *I = dyn_cast<Instruction>(V); I && !I->hasOneUse())
return false;
if (V->getValueID() != FrontV->getValueID())
return false;
if (auto *CI = dyn_cast<CmpInst>(V))
if (CI->getPredicate() != cast<CmpInst>(FrontV)->getPredicate())
return false;
if (auto *CI = dyn_cast<CastInst>(V))
if (CI->getSrcTy() != cast<CastInst>(FrontV)->getSrcTy())
return false;
if (auto *SI = dyn_cast<SelectInst>(V))
if (!isa<VectorType>(SI->getOperand(0)->getType()) ||
SI->getOperand(0)->getType() !=
cast<SelectInst>(FrontV)->getOperand(0)->getType())
return false;
if (isa<CallInst>(V) && !isa<IntrinsicInst>(V))
return false;
auto *II = dyn_cast<IntrinsicInst>(V);
return !II || (isa<IntrinsicInst>(FrontV) &&
II->getIntrinsicID() ==
cast<IntrinsicInst>(FrontV)->getIntrinsicID() &&
!II->hasOperandBundles());
};
if (all_of(drop_begin(Item), CheckLaneIsEquivalentToFirst)) {
// Check the operator is one that we support.
if (isa<BinaryOperator, CmpInst>(FrontU)) {
// We exclude div/rem in case they hit UB from poison lanes.
Expand Down Expand Up @@ -2038,7 +2040,8 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
Worklist.push_back(generateInstLaneVectorFromOperand(Item, 2));
continue;
} else if (auto *II = dyn_cast<IntrinsicInst>(FrontU);
II && isTriviallyVectorizable(II->getIntrinsicID())) {
II && isTriviallyVectorizable(II->getIntrinsicID()) &&
!II->hasOperandBundles()) {
for (unsigned Op = 0, E = II->getNumOperands() - 1; Op < E; Op++) {
if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(), Op)) {
if (!all_of(drop_begin(Item), [Item, Op](InstLane &IL) {
Expand Down
48 changes: 48 additions & 0 deletions llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1066,4 +1066,52 @@ entry:
ret <2 x float> %4
}

define <16 x i64> @operandbundles(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
; CHECK-LABEL: @operandbundles(
; CHECK-NEXT: [[CALL:%.*]] = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i64> [[C:%.*]]) [ "jl_roots"(ptr addrspace(10) null, ptr addrspace(10) null) ]
; CHECK-NEXT: [[SHUFFLEVECTOR:%.*]] = shufflevector <4 x i64> [[CALL]], <4 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[SHUFFLEVECTOR1:%.*]] = shufflevector <16 x i64> [[SHUFFLEVECTOR]], <16 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; CHECK-NEXT: ret <16 x i64> [[SHUFFLEVECTOR1]]
;
%call = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) [ "jl_roots"(ptr addrspace(10) null, ptr addrspace(10) null) ]
%shufflevector = shufflevector <4 x i64> %call, <4 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
%shufflevector1 = shufflevector <16 x i64> %shufflevector, <16 x i64> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
ret <16 x i64> %shufflevector1
}

define <8 x i8> @operandbundles_first(<8 x i8> %a) {
; CHECK-LABEL: @operandbundles_first(
; CHECK-NEXT: [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
; CHECK-NEXT: [[ABT:%.*]] = call <4 x i8> @llvm.abs.v4i8(<4 x i8> [[AT]], i1 false) [ "jl_roots"(ptr addrspace(10) null, ptr addrspace(10) null) ]
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i8> [[AT]], <4 x i8> [[AB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[R:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> [[TMP1]], i1 false)
; CHECK-NEXT: ret <8 x i8> [[R]]
;
%ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
%at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
%abt = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %at, i1 false) [ "jl_roots"(ptr addrspace(10) null, ptr addrspace(10) null) ]
%abb = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %ab, i1 false)
%r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <8 x i8> %r
}

define <8 x i8> @operandbundles_second(<8 x i8> %a) {
; CHECK-LABEL: @operandbundles_second(
; CHECK-NEXT: [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
; CHECK-NEXT: [[ABB:%.*]] = call <4 x i8> @llvm.abs.v4i8(<4 x i8> [[AB]], i1 false) [ "jl_roots"(ptr addrspace(10) null, ptr addrspace(10) null) ]
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i8> [[AT]], <4 x i8> [[AB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[R:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> [[TMP1]], i1 false)
; CHECK-NEXT: ret <8 x i8> [[R]]
;
%ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
%at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
%abt = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %at, i1 false)
%abb = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %ab, i1 false) [ "jl_roots"(ptr addrspace(10) null, ptr addrspace(10) null) ]
%r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <8 x i8> %r
}

declare <4 x i64> @llvm.fshl.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
declare void @use(<4 x i8>)

0 comments on commit c3ca15a

Please sign in to comment.