Skip to content

Commit 03934e7

Browse files
iii-iiii-i
authored
[SystemZ] Enable AtomicExpand pass (#70398)
The upcoming OpenMP support for SystemZ requires handling of IR insns like `atomicrmw fadd`. Normally atomic float operations are expanded by Clang and such insns do not occur, but OpenMP generates them directly. Other architectures handle this using the AtomicExpand pass, which SystemZ did not need so far. Enable it. Currently AtomicExpand treats atomic load and stores of floats pessimistically: it casts them to integers, which SystemZ does not need, since the floating point load and store instructions are already atomic. However, the way Clang currently expands them is pessimistic as well, so this change does not make things worse. Optimizing operations on atomic floats can be a separate change in the future. This change does not create any differences the Linux kernel build.
1 parent 03c8fbf commit 03934e7

26 files changed

+536
-0
lines changed

Diff for: ‎llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,15 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const
872872
return false;
873873
}
874874

875+
TargetLowering::AtomicExpansionKind
876+
SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
877+
return (RMW->isFloatingPointOperation() ||
878+
RMW->getOperation() == AtomicRMWInst::UIncWrap ||
879+
RMW->getOperation() == AtomicRMWInst::UDecWrap)
880+
? AtomicExpansionKind::CmpXChg
881+
: AtomicExpansionKind::None;
882+
}
883+
875884
bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
876885
// We can use CGFI or CLGFI.
877886
return isInt<32>(Imm) || isUInt<32>(Imm);

Diff for: ‎llvm/lib/Target/SystemZ/SystemZISelLowering.h

+2
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,8 @@ class SystemZTargetLowering : public TargetLowering {
452452
return VT != MVT::f64;
453453
}
454454
bool hasInlineStackProbe(const MachineFunction &MF) const override;
455+
AtomicExpansionKind
456+
shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override;
455457
bool isLegalICmpImmediate(int64_t Imm) const override;
456458
bool isLegalAddImmediate(int64_t Imm) const override;
457459
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,

Diff for: ‎llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,8 @@ void SystemZPassConfig::addIRPasses() {
226226
addPass(createLoopDataPrefetchPass());
227227
}
228228

229+
addPass(createAtomicExpandPass());
230+
229231
TargetPassConfig::addIRPasses();
230232
}
231233

Diff for: ‎llvm/test/CodeGen/SystemZ/atomic-load-06.ll

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
; Test float atomic loads.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define float @f1(ptr %src) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: lgf [[R:%r[0-9]+]], 0(%r2)
8+
; CHECK: sllg [[R]], [[R]], 32
9+
; CHECK: ldgr %f0, [[R]]
10+
; CHECK: br %r14
11+
%val = load atomic float, ptr %src seq_cst, align 4
12+
ret float %val
13+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomic-load-07.ll

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
; Test double atomic loads.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define double @f1(ptr %src) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: ld %f0, 0(%r2)
8+
; CHECK: br %r14
9+
%val = load atomic double, ptr %src seq_cst, align 8
10+
ret double %val
11+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomic-load-08.ll

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
; Test long double atomic loads. Expect a libcall.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define void @f1(ptr %ret, ptr %src) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: lgr [[RET:%r[0-9]+]], %r2
8+
; CHECK: la %r4, 160(%r15)
9+
; CHECK: lghi %r2, 16
10+
; CHECK: lhi %r5, 5
11+
; CHECK: brasl %r14, __atomic_load@PLT
12+
; CHECK: ld [[FL:%f[0-9]+]], 160(%r15)
13+
; CHECK: ld [[FH:%f[0-9]+]], 168(%r15)
14+
; CHECK: std [[FL]], 0([[RET]])
15+
; CHECK: std [[FH]], 8([[RET]])
16+
; CHECK: br %r14
17+
%val = load atomic fp128, ptr %src seq_cst, align 8
18+
store fp128 %val, ptr %ret, align 8
19+
ret void
20+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomic-store-06.ll

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
; Test float atomic loads.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define void @f1(ptr %src, float %val) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: lgdr [[R:%r[0-9]+]], %f0
8+
; CHECK: srlg [[R]], [[R]], 32
9+
; CHECK: st [[R]], 0(%r2)
10+
; CHECK: br %r14
11+
store atomic float %val, ptr %src seq_cst, align 4
12+
ret void
13+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomic-store-07.ll

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
; Test double atomic stores.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define void @f1(ptr %dst, double %val) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: std %f0, 0(%r2)
8+
; CHECK: br %r14
9+
store atomic double %val, ptr %dst seq_cst, align 8
10+
ret void
11+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomic-store-08.ll

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
; Test long double atomic stores. Expect a libcall.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define void @f1(ptr %dst, ptr %src) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: ld [[FL:%f[0-9]+]], 0(%r3)
8+
; CHECK: ld [[FH:%f[0-9]+]], 8(%r3)
9+
; CHECK: lgr %r3, %r2
10+
; CHECK: std [[FL]], 160(%r15)
11+
; CHECK: std [[FH]], 168(%r15)
12+
; CHECK: la %r4, 160(%r15)
13+
; CHECK: lghi %r2, 16
14+
; CHECK: lhi %r5, 5
15+
; CHECK: brasl %r14, __atomic_store@PLT
16+
; CHECK: br %r14
17+
%val = load fp128, ptr %src, align 8
18+
store atomic fp128 %val, ptr %dst seq_cst, align 8
19+
ret void
20+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
; Test atomic float addition. Expect a compare-and-swap loop.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define float @f1(ptr %src, float %b) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: le [[F:%f[0-9]+]], 0(%r2)
8+
; CHECK: [[L:\.L.+]]:
9+
; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
10+
; CHECK: aebr [[F]], %f0
11+
; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
12+
; CHECK: srlg [[RO]], [[RO]], 32
13+
; CHECK: srlg [[RI]], [[RI]], 32
14+
; CHECK: cs [[RI]], [[RO]], 0(%r2)
15+
; CHECK: sllg [[RI]], [[RI]], 32
16+
; CHECK: ldgr [[F]], [[RI]]
17+
; CHECK: jl [[L]]
18+
; CHECK: ler %f0, [[F]]
19+
; CHECK: br %r14
20+
%res = atomicrmw fadd ptr %src, float %b seq_cst
21+
ret float %res
22+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
; Test atomic double addition. Expect a compare-and-swap loop.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define double @f1(ptr %src, double %b) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: ld [[F:%f[0-9]+]], 0(%r2)
8+
; CHECK: [[L:\.L.+]]:
9+
; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
10+
; CHECK: adbr [[F]], %f0
11+
; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
12+
; CHECK: csg [[RI]], [[RO]], 0(%r2)
13+
; CHECK: ldgr [[F]], [[RI]]
14+
; CHECK: jl [[L]]
15+
; CHECK: ldr %f0, [[F]]
16+
; CHECK: br %r14
17+
%res = atomicrmw fadd ptr %src, double %b seq_cst
18+
ret double %res
19+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
; Test atomic long double addition. Expect a compare-and-swap loop.
2+
;
3+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
4+
5+
define void @f1(ptr %ret, ptr %src, ptr %b) {
6+
; CHECK-LABEL: f1:
7+
; CHECK: [[FBL:%f[0-9]+]], 0(%r4)
8+
; CHECK: [[FBH:%f[0-9]+]], 8(%r4)
9+
; CHECK: [[FSL:%f[0-9]+]], 0(%r3)
10+
; CHECK: [[FSH:%f[0-9]+]], 8(%r3)
11+
; CHECK: [[LABEL:\.L.+]]:
12+
; CHECK: lgdr [[RISH:%r[0-9]+]], [[FSH]]
13+
; CHECK: lgdr [[RISL:%r[0-9]+]], [[FSL]]
14+
; CHECK: axbr [[FSL]], [[FBL]]
15+
; CHECK: lgdr [[ROSH:%r[0-9]+]], [[FSH]]
16+
; CHECK: lgdr [[ROSL:%r[0-9]+]], [[FSL]]
17+
; CHECK: cdsg [[RISL]], [[ROSL]], 0(%r3)
18+
; CHECK: stg [[RISH]], 168(%r15)
19+
; CHECK: stg [[RISL]], 160(%r15)
20+
; CHECK: ld [[FSL]], 160(%r15)
21+
; CHECK: ld [[FSH]], 168(%r15)
22+
; CHECK: jl [[LABEL]]
23+
; CHECK: std [[FSL]], 0(%r2)
24+
; CHECK: std [[FSH]], 8(%r2)
25+
; CHECK: br %r14
26+
%val = load fp128, ptr %b
27+
%res = atomicrmw fadd ptr %src, fp128 %val seq_cst
28+
store fp128 %res, ptr %ret
29+
ret void
30+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
; Test atomic float maximum.
2+
; Expect a libcall in a compare-and-swap loop.
3+
;
4+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
5+
6+
define float @f1(ptr %src, float %b) {
7+
; CHECK-LABEL: f1:
8+
; CHECK: lgr [[SRC:%r[0-9]+]], %r2
9+
; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
10+
; CHECK: ler [[FB:%f[0-9]+]], %f0
11+
; CHECK: [[L:\.L.+]]:
12+
; CHECK: ler %f0, [[FSRC]]
13+
; CHECK: ler %f2, [[FB]]
14+
; CHECK: brasl %r14, fmaxf@PLT
15+
; CHECK: lgdr [[RO:%r[0-9]+]], %f0
16+
; CHECK: srlg [[RO]], [[RO]], 32
17+
; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
18+
; CHECK: srlg [[RI]], [[RI]], 32
19+
; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
20+
; CHECK: sllg [[RO]], [[RI]], 32
21+
; CHECK: ldgr [[FSRC]], [[RO]]
22+
; CHECK: jl [[L]]
23+
; CHECK: ler %f0, [[FSRC]]
24+
; CHECK: br %r14
25+
%res = atomicrmw fmax ptr %src, float %b seq_cst
26+
ret float %res
27+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
; Test atomic double maximum.
2+
; Expect a libcall in a compare-and-swap loop.
3+
;
4+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
5+
6+
define double @f1(ptr %src, double %b) {
7+
; CHECK-LABEL: f1:
8+
; CHECK: lgr [[RB:%r[0-9]+]], %r2
9+
; CHECK: ld [[FB:%f[0-9]+]], 0(%r2)
10+
; CHECK: ldr [[FSRC:%f[0-9]+]], %f0
11+
; CHECK: [[L:\.L.+]]:
12+
; CHECK: ldr %f0, [[FB]]
13+
; CHECK: ldr %f2, [[FSRC]]
14+
; CHECK: brasl %r14, fmax@PLT
15+
; CHECK: lgdr [[RO:%r[0-9]+]], %f0
16+
; CHECK: lgdr [[RI:%r[0-9]+]], [[FB]]
17+
; CHECK: csg [[RI]], [[RO]], 0([[RB]])
18+
; CHECK: ldgr [[FB]], [[RI]]
19+
; CHECK: jl [[L]]
20+
; CHECK: ldr %f0, [[FB]]
21+
; CHECK: br %r14
22+
%res = atomicrmw fmax ptr %src, double %b seq_cst
23+
ret double %res
24+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
; Test atomic long double maximum.
2+
; Expect a libcall in a compare-and-swap loop.
3+
;
4+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
5+
6+
define void @f1(ptr %ret, ptr %src, ptr %b) {
7+
; CHECK-LABEL: f1:
8+
; CHECK: lgr [[SRC:%r[0-9]+]], %r3
9+
; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
10+
; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
11+
; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
12+
; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
13+
; CHECK: lgr [[RET:%r[0-9]+]], %r2
14+
; CHECK: [[L:\.L.+]]:
15+
; CHECK: std [[FBL]], 160(%r15)
16+
; CHECK: std [[FBH]], 168(%r15)
17+
; CHECK: la %r2, 192(%r15)
18+
; CHECK: la %r3, 176(%r15)
19+
; CHECK: la %r4, 160(%r15)
20+
; CHECK: std [[FSL]], 176(%r15)
21+
; CHECK: std [[FSH]], 184(%r15)
22+
; CHECK: brasl %r14, fmaxl@PLT
23+
; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
24+
; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
25+
; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
26+
; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
27+
; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
28+
; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
29+
; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
30+
; CHECK: stg [[RSH]], 216(%r15)
31+
; CHECK: stg [[RSL]], 208(%r15)
32+
; CHECK: ld [[FSL]], 208(%r15)
33+
; CHECK: ld [[FSH]], 216(%r15)
34+
; CHECK: jl [[L]]
35+
; CHECK: std [[FSL]], 0([[RET]])
36+
; CHECK: std [[FSH]], 8([[RET]])
37+
; CHECK: br %r14
38+
%val = load fp128, ptr %b
39+
%res = atomicrmw fmax ptr %src, fp128 %val seq_cst
40+
store fp128 %res, ptr %ret
41+
ret void
42+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
; Test atomic float minimum.
2+
; Expect a libcall in a compare-and-swap loop.
3+
;
4+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
5+
6+
define float @f1(ptr %src, float %b) {
7+
; CHECK-LABEL: f1:
8+
; CHECK: lgr [[SRC:%r[0-9]+]], %r2
9+
; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
10+
; CHECK: ler [[FB:%f[0-9]+]], %f0
11+
; CHECK: [[L:\.L.+]]:
12+
; CHECK: ler %f0, [[FSRC]]
13+
; CHECK: ler %f2, [[FB]]
14+
; CHECK: brasl %r14, fminf@PLT
15+
; CHECK: lgdr [[RO:%r[0-9]+]], %f0
16+
; CHECK: srlg [[RO]], [[RO]], 32
17+
; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
18+
; CHECK: srlg [[RI]], [[RI]], 32
19+
; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
20+
; CHECK: sllg [[RO]], [[RI]], 32
21+
; CHECK: ldgr [[FSRC]], [[RO]]
22+
; CHECK: jl [[L]]
23+
; CHECK: ler %f0, [[FSRC]]
24+
; CHECK: br %r14
25+
%res = atomicrmw fmin ptr %src, float %b seq_cst
26+
ret float %res
27+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
; Test atomic double minimum.
2+
; Expect a libcall in a compare-and-swap loop.
3+
;
4+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
5+
6+
define double @f1(ptr %src, double %b) {
7+
; CHECK-LABEL: f1:
8+
; CHECK: lgr [[SRC:%r[0-9]+]], %r2
9+
; CHECK: ld [[FSRC:%f[0-9]+]], 0(%r2)
10+
; CHECK: ldr [[FB:%f[0-9]+]], %f0
11+
; CHECK: [[L:\.L.+]]:
12+
; CHECK: ldr %f0, [[FSRC]]
13+
; CHECK: ldr %f2, [[FB]]
14+
; CHECK: brasl %r14, fmin@PLT
15+
; CHECK: lgdr [[RO:%r[0-9]+]], %f0
16+
; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
17+
; CHECK: csg [[RI]], [[RO]], 0([[SRC]])
18+
; CHECK: ldgr [[FSRC]], [[RI]]
19+
; CHECK: jl [[L]]
20+
; CHECK: ldr %f0, [[FSRC]]
21+
; CHECK: br %r14
22+
%res = atomicrmw fmin ptr %src, double %b seq_cst
23+
ret double %res
24+
}

Diff for: ‎llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
; Test atomic long double minimum.
2+
; Expect a libcall in a compare-and-swap loop.
3+
;
4+
; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
5+
6+
define void @f1(ptr %ret, ptr %src, ptr %b) {
7+
; CHECK-LABEL: f1:
8+
; CHECK: lgr [[SRC:%r[0-9]+]], %r3
9+
; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
10+
; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
11+
; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
12+
; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
13+
; CHECK: lgr [[RET:%r[0-9]+]], %r2
14+
; CHECK: [[L:\.L.+]]:
15+
; CHECK: std [[FBL]], 160(%r15)
16+
; CHECK: std [[FBH]], 168(%r15)
17+
; CHECK: la %r2, 192(%r15)
18+
; CHECK: la %r3, 176(%r15)
19+
; CHECK: la %r4, 160(%r15)
20+
; CHECK: std [[FSL]], 176(%r15)
21+
; CHECK: std [[FSH]], 184(%r15)
22+
; CHECK: brasl %r14, fminl@PLT
23+
; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
24+
; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
25+
; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
26+
; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
27+
; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
28+
; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
29+
; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
30+
; CHECK: stg [[RSH]], 216(%r15)
31+
; CHECK: stg [[RSL]], 208(%r15)
32+
; CHECK: ld [[FSL]], 208(%r15)
33+
; CHECK: ld [[FSH]], 216(%r15)
34+
; CHECK: jl [[L]]
35+
; CHECK: std [[FSL]], 0([[RET]])
36+
; CHECK: std [[FSH]], 8([[RET]])
37+
; CHECK: br %r14
38+
%val = load fp128, ptr %b
39+
%res = atomicrmw fmin ptr %src, fp128 %val seq_cst
40+
store fp128 %res, ptr %ret
41+
ret void
42+
}

0 commit comments

Comments
 (0)