Skip to content

Commit d81ffd4

Browse files
alexey-bataevalexey-bataev
committed
[SLP]INsert postponed vector value after all uses, if the parent node is PHI
Need to insert the vector value for the postponed gather/buildvector node after all uses non only if the vector value of the user node is phi, but also if the user node itself is PHI node, which may produce vector phi + shuffle. Fixes #162799
1 parent 5e3ac2a commit d81ffd4

File tree

3 files changed

+123
-3
lines changed

3 files changed

+123
-3
lines changed

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20076,7 +20076,9 @@ Value *BoUpSLP::vectorizeTree(
2007620076
// The is because source vector that supposed to feed this gather node was
2007720077
// inserted at the end of the block [after stab instruction]. So we need
2007820078
// to adjust insertion point again to the end of block.
20079-
if (isa<PHINode>(UserI)) {
20079+
if (isa<PHINode>(UserI) ||
20080+
(TE->UserTreeIndex.UserTE->hasState() &&
20081+
TE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI)) {
2008020082
// Insert before all users.
2008120083
Instruction *InsertPt = PrevVec->getParent()->getTerminator();
2008220084
for (User *U : PrevVec->users()) {

‎llvm/test/Transforms/SLPVectorizer/X86/parent-phi-node-reordered.ll‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
2+
; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
3+
4+
define void @test(i32 %arg, i32 %arg1) {
5+
; CHECK-LABEL: define void @test(
6+
; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) {
7+
; CHECK-NEXT: [[BB:.*]]:
8+
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[ARG1]], i32 0
9+
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[ARG]], i32 0
10+
; CHECK-NEXT: br label %[[BB6:.*]]
11+
; CHECK: [[BB2:.*]]:
12+
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ [[TMP14:%.*]], %[[BB19:.*]] ]
13+
; CHECK-NEXT: ret void
14+
; CHECK: [[BB6]]:
15+
; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP17:%.*]], %[[BB26:.*]] ], [ [[TMP16:%.*]], %[[BB27:.*]] ], [ zeroinitializer, %[[BB25:.*]] ]
16+
; CHECK-NEXT: switch i8 0, label %[[BB11:.*]] [
17+
; CHECK-NEXT: i8 0, label %[[BB28:.*]]
18+
; CHECK-NEXT: ]
19+
; CHECK: [[BB11]]:
20+
; CHECK-NEXT: [[PHI12:%.*]] = phi i32 [ 0, %[[BB28]] ], [ 0, %[[BB6]] ]
21+
; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i32> [ [[TMP3]], %[[BB28]] ], [ zeroinitializer, %[[BB6]] ]
22+
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> <i32 poison, i32 0, i32 poison, i32 poison>, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
23+
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[ARG]], i32 0
24+
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[PHI12]], i32 3
25+
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
26+
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP1]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
27+
; CHECK-NEXT: switch i8 0, label %[[BB19]] [
28+
; CHECK-NEXT: i8 1, label %[[BB17:.*]]
29+
; CHECK-NEXT: i8 0, label %[[BB18:.*]]
30+
; CHECK-NEXT: ]
31+
; CHECK: [[BB17]]:
32+
; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], zeroinitializer
33+
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <4 x i32> <i32 0, i32 3, i32 6, i32 poison>
34+
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
35+
; CHECK-NEXT: br label %[[BB19]]
36+
; CHECK: [[BB18]]:
37+
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 2, i32 0>
38+
; CHECK-NEXT: br label %[[BB19]]
39+
; CHECK: [[BB19]]:
40+
; CHECK-NEXT: [[TMP14]] = phi <4 x i32> [ [[TMP10]], %[[BB17]] ], [ [[TMP7]], %[[BB18]] ], [ [[TMP9]], %[[BB11]] ]
41+
; CHECK-NEXT: [[TMP15:%.*]] = phi <4 x i32> [ [[TMP12]], %[[BB17]] ], [ [[TMP13]], %[[BB18]] ], [ [[TMP7]], %[[BB11]] ]
42+
; CHECK-NEXT: [[TMP16]] = shufflevector <4 x i32> [[TMP15]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
43+
; CHECK-NEXT: br i1 false, label %[[BB2]], label %[[BB25]]
44+
; CHECK: [[BB25]]:
45+
; CHECK-NEXT: switch i8 0, label %[[BB6]] [
46+
; CHECK-NEXT: i8 0, label %[[BB26]]
47+
; CHECK-NEXT: i8 1, label %[[BB27]]
48+
; CHECK-NEXT: i8 6, label %[[BB27]]
49+
; CHECK-NEXT: ]
50+
; CHECK: [[BB26]]:
51+
; CHECK-NEXT: [[TMP17]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> [[TMP0]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
52+
; CHECK-NEXT: br label %[[BB6]]
53+
; CHECK: [[BB27]]:
54+
; CHECK-NEXT: br label %[[BB6]]
55+
; CHECK: [[BB28]]:
56+
; CHECK-NEXT: br label %[[BB11]]
57+
;
58+
bb:
59+
br label %bb6
60+
61+
bb2:
62+
%phi = phi i32 [ %phi21, %bb19 ]
63+
%phi3 = phi i32 [ %phi22, %bb19 ]
64+
%phi4 = phi i32 [ %phi23, %bb19 ]
65+
%phi5 = phi i32 [ %phi24, %bb19 ]
66+
ret void
67+
68+
bb6:
69+
%phi7 = phi i32 [ 0, %bb ], [ %phi24, %bb26 ], [ %phi24, %bb27 ], [ 0, %bb25 ]
70+
%phi8 = phi i32 [ 0, %bb ], [ %arg1, %bb26 ], [ %phi23, %bb27 ], [ 0, %bb25 ]
71+
%phi9 = phi i32 [ 0, %bb ], [ %phi22, %bb26 ], [ %phi20, %bb27 ], [ 0, %bb25 ]
72+
%phi10 = phi i32 [ 0, %bb ], [ %phi21, %bb26 ], [ %phi21, %bb27 ], [ 0, %bb25 ]
73+
switch i8 0, label %bb11 [
74+
i8 0, label %bb28
75+
]
76+
77+
bb11:
78+
%phi12 = phi i32 [ 0, %bb28 ], [ 0, %bb6 ]
79+
%phi13 = phi i32 [ %phi10, %bb28 ], [ 0, %bb6 ]
80+
%phi14 = phi i32 [ %phi9, %bb28 ], [ 0, %bb6 ]
81+
%phi15 = phi i32 [ %phi8, %bb28 ], [ 0, %bb6 ]
82+
%phi16 = phi i32 [ %phi7, %bb28 ], [ 0, %bb6 ]
83+
switch i8 0, label %bb19 [
84+
i8 1, label %bb17
85+
i8 0, label %bb18
86+
]
87+
88+
bb17:
89+
%add = add i32 %phi16, 0
90+
br label %bb19
91+
92+
bb18:
93+
br label %bb19
94+
95+
bb19:
96+
%phi20 = phi i32 [ 0, %bb17 ], [ %arg, %bb18 ], [ %phi12, %bb11 ]
97+
%phi21 = phi i32 [ %phi13, %bb17 ], [ %phi12, %bb18 ], [ 0, %bb11 ]
98+
%phi22 = phi i32 [ %phi14, %bb17 ], [ 0, %bb18 ], [ 0, %bb11 ]
99+
%phi23 = phi i32 [ %phi15, %bb17 ], [ %arg, %bb18 ], [ %arg, %bb11 ]
100+
%phi24 = phi i32 [ %add, %bb17 ], [ %phi16, %bb18 ], [ %phi16, %bb11 ]
101+
br i1 false, label %bb2, label %bb25
102+
103+
bb25:
104+
switch i8 0, label %bb6 [
105+
i8 0, label %bb26
106+
i8 1, label %bb27
107+
i8 6, label %bb27
108+
]
109+
110+
bb26:
111+
br label %bb6
112+
113+
bb27:
114+
br label %bb6
115+
116+
bb28:
117+
br label %bb11
118+
}

‎llvm/test/Transforms/SLPVectorizer/X86/phi-nodes-incoming-same-blocks.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,17 @@ define void @test(ptr %0, i1 %1, i1 %2) {
66
; CHECK-SAME: ptr [[TMP0:%.*]], i1 [[TMP1:%.*]], i1 [[TMP2:%.*]]) #[[ATTR0:[0-9]+]] {
77
; CHECK-NEXT: br label %[[BB4:.*]]
88
; CHECK: [[BB4]]:
9-
; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ [[TMP12:%.*]], %[[TMP7:.*]] ], [ zeroinitializer, [[TMP3:%.*]] ]
9+
; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ [[TMP15:%.*]], %[[TMP7:.*]] ], [ zeroinitializer, [[TMP3:%.*]] ]
1010
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
1111
; CHECK-NEXT: br i1 [[TMP1]], label %[[TMP7]], label %[[BB15:.*]]
1212
; CHECK: [[TMP7]]:
1313
; CHECK-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP0]], align 8
1414
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 16
1515
; CHECK-NEXT: [[TMP10:%.*]] = load <2 x i32>, ptr [[TMP9]], align 1
1616
; CHECK-NEXT: [[TMP11:%.*]] = or <2 x i32> [[TMP10]], splat (i32 1)
17-
; CHECK-NEXT: [[TMP12]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> <i32 1, i32 poison>, <2 x i32> <i32 2, i32 1>
1817
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
1918
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
19+
; CHECK-NEXT: [[TMP15]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> <i32 1, i32 poison>, <2 x i32> <i32 2, i32 1>
2020
; CHECK-NEXT: br i1 [[TMP2]], label %[[BB16:.*]], label %[[BB4]]
2121
; CHECK: [[BB15]]:
2222
; CHECK-NEXT: br label %[[BB16]]

0 commit comments

Comments
 (0)