diff --git a/src/runtime/race.go b/src/runtime/race.go index ca4f0519799022..a3dff30c861203 100644 --- a/src/runtime/race.go +++ b/src/runtime/race.go @@ -323,6 +323,10 @@ var __tsan_report_count byte //go:cgo_import_static __tsan_go_atomic64_exchange //go:cgo_import_static __tsan_go_atomic32_fetch_add //go:cgo_import_static __tsan_go_atomic64_fetch_add +//go:cgo_import_static __tsan_go_atomic32_fetch_and +//go:cgo_import_static __tsan_go_atomic64_fetch_and +//go:cgo_import_static __tsan_go_atomic32_fetch_or +//go:cgo_import_static __tsan_go_atomic64_fetch_or //go:cgo_import_static __tsan_go_atomic32_compare_exchange //go:cgo_import_static __tsan_go_atomic64_compare_exchange @@ -642,6 +646,36 @@ func abigen_sync_atomic_AddUint64(addr *uint64, delta uint64) (new uint64) //go:linkname abigen_sync_atomic_AddUintptr sync/atomic.AddUintptr func abigen_sync_atomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr) +//go:linkname abigen_sync_atomic_AndInt32 sync/atomic.AndInt32 +func abigen_sync_atomic_AndInt32(addr *int32, mask int32) (old int32) + +//go:linkname abigen_sync_atomic_AndUint32 sync/atomic.AndUint32 +func abigen_sync_atomic_AndUint32(addr *uint32, mask uint32) (old uint32) + +//go:linkname abigen_sync_atomic_AndInt64 sync/atomic.AndInt64 +func abigen_sync_atomic_AndInt64(addr *int64, mask int64) (old int64) + +//go:linkname abigen_sync_atomic_AndUint64 sync/atomic.AndUint64 +func abigen_sync_atomic_AndUint64(addr *uint64, mask uint64) (old uint64) + +//go:linkname abigen_sync_atomic_AndUintptr sync/atomic.AndUintptr +func abigen_sync_atomic_AndUintptr(addr *uintptr, mask uintptr) (old uintptr) + +//go:linkname abigen_sync_atomic_OrInt32 sync/atomic.OrInt32 +func abigen_sync_atomic_OrInt32(addr *int32, mask int32) (old int32) + +//go:linkname abigen_sync_atomic_OrUint32 sync/atomic.OrUint32 +func abigen_sync_atomic_OrUint32(addr *uint32, mask uint32) (old uint32) + +//go:linkname abigen_sync_atomic_OrInt64 sync/atomic.OrInt64 +func abigen_sync_atomic_OrInt64(addr *int64, mask int64) (old int64) + +//go:linkname abigen_sync_atomic_OrUint64 sync/atomic.OrUint64 +func abigen_sync_atomic_OrUint64(addr *uint64, mask uint64) (old uint64) + +//go:linkname abigen_sync_atomic_OrUintptr sync/atomic.OrUintptr +func abigen_sync_atomic_OrUintptr(addr *uintptr, mask uintptr) (old uintptr) + //go:linkname abigen_sync_atomic_CompareAndSwapInt32 sync/atomic.CompareAndSwapInt32 func abigen_sync_atomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool) diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index 45c1255509c8fb..15f29cca4cee70 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -303,6 +303,56 @@ TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24 GO_ARGS JMP sync∕atomic·AddInt64(SB) +// And +TEXT sync∕atomic·AndInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVQ $__tsan_go_atomic32_fetch_and(SB), AX + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVQ $__tsan_go_atomic64_fetch_and(SB), AX + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AndInt32(SB) + +TEXT sync∕atomic·AndUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +TEXT sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +// Or +TEXT sync∕atomic·OrInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVQ $__tsan_go_atomic32_fetch_or(SB), AX + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVQ $__tsan_go_atomic64_fetch_or(SB), AX + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·OrInt32(SB) + +TEXT sync∕atomic·OrUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +TEXT sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + // CompareAndSwap TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT|NOFRAME, $0-17 GO_ARGS diff --git a/src/runtime/race_arm64.s b/src/runtime/race_arm64.s index c818345852f6b0..6bc5eded6a8230 100644 --- a/src/runtime/race_arm64.s +++ b/src/runtime/race_arm64.s @@ -312,6 +312,56 @@ TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24 GO_ARGS JMP sync∕atomic·AddInt64(SB) +// And +TEXT sync∕atomic·AndInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVD $__tsan_go_atomic32_fetch_and(SB), R9 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVD $__tsan_go_atomic64_fetch_and(SB), R9 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AndInt32(SB) + +TEXT sync∕atomic·AndUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +TEXT sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +// Or +TEXT sync∕atomic·OrInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVD $__tsan_go_atomic32_fetch_or(SB), R9 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVD $__tsan_go_atomic64_fetch_or(SB), R9 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·OrInt32(SB) + +TEXT sync∕atomic·OrUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +TEXT sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + // CompareAndSwap TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17 GO_ARGS diff --git a/src/runtime/race_ppc64le.s b/src/runtime/race_ppc64le.s index 39cfffc39bbeb7..ef3721b6680196 100644 --- a/src/runtime/race_ppc64le.s +++ b/src/runtime/race_ppc64le.s @@ -325,6 +325,56 @@ TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24 GO_ARGS BR sync∕atomic·AddInt64(SB) +// And +TEXT sync∕atomic·AndInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVD $__tsan_go_atomic32_fetch_and(SB), R8 + BR racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVD $__tsan_go_atomic64_fetch_and(SB), R8 + BR racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndUint32(SB), NOSPLIT, $0-20 + GO_ARGS + BR sync∕atomic·AndInt32(SB) + +TEXT sync∕atomic·AndUint64(SB), NOSPLIT, $0-24 + GO_ARGS + BR sync∕atomic·AndInt64(SB) + +TEXT sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + BR sync∕atomic·AndInt64(SB) + +// Or +TEXT sync∕atomic·OrInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVD $__tsan_go_atomic32_fetch_or(SB), R8 + BR racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVD $__tsan_go_atomic64_fetch_or(SB), R8 + BR racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrUint32(SB), NOSPLIT, $0-20 + GO_ARGS + BR sync∕atomic·OrInt32(SB) + +TEXT sync∕atomic·OrUint64(SB), NOSPLIT, $0-24 + GO_ARGS + BR sync∕atomic·OrInt64(SB) + +TEXT sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + BR sync∕atomic·OrInt64(SB) + // CompareAndSwap in tsan TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17 GO_ARGS diff --git a/src/runtime/race_s390x.s b/src/runtime/race_s390x.s index dadc12f4dbfba5..6f59561bf912c3 100644 --- a/src/runtime/race_s390x.s +++ b/src/runtime/race_s390x.s @@ -274,6 +274,56 @@ TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24 GO_ARGS JMP sync∕atomic·AddInt64(SB) +// And +TEXT sync∕atomic·AndInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVD $__tsan_go_atomic32_fetch_and(SB), R1 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVD $__tsan_go_atomic64_fetch_and(SB), R1 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AndInt32(SB) + +TEXT sync∕atomic·AndUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +TEXT sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +// Or +TEXT sync∕atomic·OrInt32(SB), NOSPLIT|NOFRAME, $0-20 + GO_ARGS + MOVD $__tsan_go_atomic32_fetch_or(SB), R1 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrInt64(SB), NOSPLIT|NOFRAME, $0-24 + GO_ARGS + MOVD $__tsan_go_atomic64_fetch_or(SB), R1 + BL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·OrInt32(SB) + +TEXT sync∕atomic·OrUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +TEXT sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + // CompareAndSwap TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17