Skip to content

Commit

Permalink
cmd/compile: make sync/atomic AND/OR operations intrinsic on amd64
Browse files Browse the repository at this point in the history 
Update #61395

Change-Id: I59a950f48efc587dfdffce00e2f4f3ab99d8df00
Reviewed-on: https://go-review.googlesource.com/c/go/+/594738
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Nicolas Hillegeer <aktau@google.com>
  • Loading branch information
@randall77
randall77 committed Jul 23, 2024
1 parent dbfa3ca commit c18ff29
Show file tree
Hide file tree
Showing 8 changed files with 478 additions and 14 deletions.
57 changes: 56 additions & 1 deletion src/cmd/compile/internal/amd64/ssa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1286,14 +1286,69 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p = s.Prog(x86.ASETEQ)
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg0()
case ssa.OpAMD64ANDBlock, ssa.OpAMD64ANDLlock, ssa.OpAMD64ORBlock, ssa.OpAMD64ORLlock:
case ssa.OpAMD64ANDBlock, ssa.OpAMD64ANDLlock, ssa.OpAMD64ANDQlock, ssa.OpAMD64ORBlock, ssa.OpAMD64ORLlock, ssa.OpAMD64ORQlock:
// Atomic memory operations that don't need to return the old value.
s.Prog(x86.ALOCK)
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = v.Args[1].Reg()
p.To.Type = obj.TYPE_MEM
p.To.Reg = v.Args[0].Reg()
ssagen.AddAux(&p.To, v)
case ssa.OpAMD64LoweredAtomicAnd64, ssa.OpAMD64LoweredAtomicOr64, ssa.OpAMD64LoweredAtomicAnd32, ssa.OpAMD64LoweredAtomicOr32:
// Atomic memory operations that need to return the old value.
// We need to do these with compare-and-exchange to get access to the old value.
// loop:
// MOVQ mask, tmp
// MOVQ (addr), AX
// ANDQ AX, tmp
// LOCK CMPXCHGQ tmp, (addr) : note that AX is implicit old value to compare against
// JNE loop
// : result in AX
mov := x86.AMOVQ
op := x86.AANDQ
cmpxchg := x86.ACMPXCHGQ
switch v.Op {
case ssa.OpAMD64LoweredAtomicOr64:
op = x86.AORQ
case ssa.OpAMD64LoweredAtomicAnd32:
mov = x86.AMOVL
op = x86.AANDL
cmpxchg = x86.ACMPXCHGL
case ssa.OpAMD64LoweredAtomicOr32:
mov = x86.AMOVL
op = x86.AORL
cmpxchg = x86.ACMPXCHGL
}
addr := v.Args[0].Reg()
mask := v.Args[1].Reg()
tmp := v.RegTmp()
p1 := s.Prog(mov)
p1.From.Type = obj.TYPE_REG
p1.From.Reg = mask
p1.To.Type = obj.TYPE_REG
p1.To.Reg = tmp
p2 := s.Prog(mov)
p2.From.Type = obj.TYPE_MEM
p2.From.Reg = addr
ssagen.AddAux(&p2.From, v)
p2.To.Type = obj.TYPE_REG
p2.To.Reg = x86.REG_AX
p3 := s.Prog(op)
p3.From.Type = obj.TYPE_REG
p3.From.Reg = x86.REG_AX
p3.To.Type = obj.TYPE_REG
p3.To.Reg = tmp
s.Prog(x86.ALOCK)
p5 := s.Prog(cmpxchg)
p5.From.Type = obj.TYPE_REG
p5.From.Reg = tmp
p5.To.Type = obj.TYPE_MEM
p5.To.Reg = addr
ssagen.AddAux(&p5.To, v)
p6 := s.Prog(x86.AJNE)
p6.To.Type = obj.TYPE_BRANCH
p6.To.SetTarget(p1)
case ssa.OpAMD64PrefetchT0, ssa.OpAMD64PrefetchNTA:
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_MEM
Expand Down
8 changes: 7 additions & 1 deletion src/cmd/compile/internal/ssa/_gen/AMD64.rules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,12 +578,15 @@
(AtomicCompareAndSwap32 ptr old new_ mem) => (CMPXCHGLlock ptr old new_ mem)
(AtomicCompareAndSwap64 ptr old new_ mem) => (CMPXCHGQlock ptr old new_ mem)

// Atomic memory updates.
// Atomic memory logical operations (old style).
(AtomicAnd8 ptr val mem) => (ANDBlock ptr val mem)
(AtomicAnd32 ptr val mem) => (ANDLlock ptr val mem)
(AtomicOr8 ptr val mem) => (ORBlock ptr val mem)
(AtomicOr32 ptr val mem) => (ORLlock ptr val mem)

// Atomic memory logical operations (new style).
(Atomic(And64|And32|Or64|Or32)value ptr val mem) => (LoweredAtomic(And64|And32|Or64|Or32) ptr val mem)

// Write barrier.
(WB ...) => (LoweredWB ...)

Expand Down Expand Up @@ -1697,3 +1700,6 @@
((SHL|SHR|SAR)XQload [off] {sym} ptr (MOVQconst [c]) mem) => ((SHL|SHR|SAR)Qconst [int8(c&63)] (MOVQload [off] {sym} ptr mem))
((SHL|SHR|SAR)XQload [off] {sym} ptr (MOVLconst [c]) mem) => ((SHL|SHR|SAR)Qconst [int8(c&63)] (MOVQload [off] {sym} ptr mem))
((SHL|SHR|SAR)XLload [off] {sym} ptr (MOVLconst [c]) mem) => ((SHL|SHR|SAR)Lconst [int8(c&31)] (MOVLload [off] {sym} ptr mem))

// Convert atomic logical operations to easier ones if we don't use the result.
(Select1 a:(LoweredAtomic(And64|And32|Or64|Or32) ptr val mem)) && a.Uses == 1 && clobber(a) => ((ANDQ|ANDL|ORQ|ORL)lock ptr val mem)
14 changes: 13 additions & 1 deletion src/cmd/compile/internal/ssa/_gen/AMD64Ops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ func init() {
gpstoreconstidx = regInfo{inputs: []regMask{gpspsbg, gpsp, 0}}
gpstorexchg = regInfo{inputs: []regMask{gp, gpspsbg, 0}, outputs: []regMask{gp}}
cmpxchg = regInfo{inputs: []regMask{gp, ax, gp, 0}, outputs: []regMask{gp, 0}, clobbers: ax}
atomicLogic = regInfo{inputs: []regMask{gp &^ ax, gp &^ ax, 0}, outputs: []regMask{ax, 0}}

fp01 = regInfo{inputs: nil, outputs: fponly}
fp21 = regInfo{inputs: []regMask{fp, fp}, outputs: fponly}
Expand Down Expand Up @@ -1040,11 +1041,22 @@ func init() {
{name: "CMPXCHGLlock", argLength: 4, reg: cmpxchg, asm: "CMPXCHGL", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"},
{name: "CMPXCHGQlock", argLength: 4, reg: cmpxchg, asm: "CMPXCHGQ", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"},

// Atomic memory updates.
// Atomic memory updates using logical operations.
// Old style that just returns the memory state.
{name: "ANDBlock", argLength: 3, reg: gpstore, asm: "ANDB", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, // *(arg0+auxint+aux) &= arg1
{name: "ANDLlock", argLength: 3, reg: gpstore, asm: "ANDL", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, // *(arg0+auxint+aux) &= arg1
{name: "ANDQlock", argLength: 3, reg: gpstore, asm: "ANDQ", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, // *(arg0+auxint+aux) &= arg1
{name: "ORBlock", argLength: 3, reg: gpstore, asm: "ORB", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, // *(arg0+auxint+aux) |= arg1
{name: "ORLlock", argLength: 3, reg: gpstore, asm: "ORL", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, // *(arg0+auxint+aux) |= arg1
{name: "ORQlock", argLength: 3, reg: gpstore, asm: "ORQ", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr"}, // *(arg0+auxint+aux) |= arg1

// Atomic memory updates using logical operations.
// *(arg0+auxint+aux) op= arg1. arg2=mem.
// New style that returns a tuple of <old contents of *(arg0+auxint+aux), memory>.
{name: "LoweredAtomicAnd64", argLength: 3, reg: atomicLogic, resultNotInArgs: true, asm: "ANDQ", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr", unsafePoint: true, needIntTemp: true},
{name: "LoweredAtomicAnd32", argLength: 3, reg: atomicLogic, resultNotInArgs: true, asm: "ANDL", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr", unsafePoint: true, needIntTemp: true},
{name: "LoweredAtomicOr64", argLength: 3, reg: atomicLogic, resultNotInArgs: true, asm: "ORQ", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr", unsafePoint: true, needIntTemp: true},
{name: "LoweredAtomicOr32", argLength: 3, reg: atomicLogic, resultNotInArgs: true, asm: "ORL", aux: "SymOff", clobberFlags: true, faultOnNilArg0: true, hasSideEffects: true, symEffect: "RdWr", unsafePoint: true, needIntTemp: true},

// Prefetch instructions
// Do prefetch arg0 address. arg0=addr, arg1=memory. Instruction variant selects locality hint
Expand Down
130 changes: 130 additions & 0 deletions src/cmd/compile/internal/ssa/opGen.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 10 additions & 1 deletion src/cmd/compile/internal/ssa/regalloc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1612,8 +1612,14 @@ func (s *regAllocState) regalloc(f *Func) {
// allocate it after all the input registers, but before
// the input registers are freed via advanceUses below.
// (Not all instructions need that distinct part, but it is conservative.)
// We also ensure it is not any of the single-choice output registers.
if opcodeTable[v.Op].needIntTemp {
m := s.allocatable & s.f.Config.gpRegMask
for _, out := range regspec.outputs {
if countRegs(out.regs) == 1 {
m &^= out.regs
}
}
if m&^desired.avoid&^s.nospill != 0 {
m &^= desired.avoid
}
Expand Down Expand Up @@ -1651,9 +1657,12 @@ func (s *regAllocState) regalloc(f *Func) {
used |= regMask(1) << tmpReg
}
for _, out := range regspec.outputs {
if out.regs == 0 {
continue
}
mask := out.regs & s.allocatable &^ used
if mask == 0 {
continue
s.f.Fatalf("can't find any output register %s", v.LongString())
}
if opcodeTable[v.Op].resultInArg0 && out.idx == 0 {
if !opcodeTable[v.Op].commutative {
Expand Down
Loading

0 comments on commit c18ff29

Please sign in to comment.