sync: allow inlining the RWMutex.RUnlock fast path

CAFxX · bradfitz · commit 05051b56a018 · 2019-03-09T16:34:17.000Z
RWMutex.RLock is already inlineable, so add a test for it as well. name old time/op new time/op delta RWMutexUncontended 66.5ns ± 0% 60.3ns ± 1% -9.38% (p=0.000 n=12+20) RWMutexUncontended-4 16.7ns ± 0% 15.3ns ± 1% -8.49% (p=0.000 n=17+20) RWMutexUncontended-16 7.86ns ± 0% 7.69ns ± 0% -2.08% (p=0.000 n=18+15) RWMutexWrite100 25.1ns ± 0% 24.0ns ± 1% -4.28% (p=0.000 n=20+18) RWMutexWrite100-4 46.7ns ± 5% 44.1ns ± 4% -5.53% (p=0.000 n=20+20) RWMutexWrite100-16 68.3ns ±11% 65.7ns ± 8% -3.81% (p=0.003 n=20+20) RWMutexWrite10 26.7ns ± 1% 25.7ns ± 0% -3.75% (p=0.000 n=17+14) RWMutexWrite10-4 34.9ns ± 2% 33.8ns ± 2% -3.15% (p=0.000 n=20+20) RWMutexWrite10-16 37.4ns ± 2% 36.1ns ± 2% -3.51% (p=0.000 n=18+20) RWMutexWorkWrite100 163ns ± 0% 162ns ± 0% -0.89% (p=0.000 n=18+20) RWMutexWorkWrite100-4 189ns ± 4% 184ns ± 4% -2.89% (p=0.000 n=19+20) RWMutexWorkWrite100-16 207ns ± 4% 200ns ± 2% -3.07% (p=0.000 n=19+20) RWMutexWorkWrite10 153ns ± 0% 151ns ± 1% -0.75% (p=0.000 n=20+20) RWMutexWorkWrite10-4 177ns ± 1% 176ns ± 2% -0.63% (p=0.004 n=17+20) RWMutexWorkWrite10-16 191ns ± 2% 189ns ± 1% -0.83% (p=0.015 n=20+17) linux/amd64 bin/go 14688201 (previous commit 14675861, +12340/+0.08%) The cumulative effect of this and the previous 3 commits is: name old time/op new time/op delta MutexUncontended 19.3ns ± 1% 16.4ns ± 1% -15.13% (p=0.000 n=20+20) MutexUncontended-4 5.24ns ± 0% 4.09ns ± 0% -21.95% (p=0.000 n=20+18) MutexUncontended-16 2.10ns ± 0% 2.12ns ± 0% +0.95% (p=0.000 n=15+17) Mutex 19.6ns ± 0% 16.3ns ± 1% -17.12% (p=0.000 n=20+20) Mutex-4 54.6ns ± 5% 45.6ns ±10% -16.51% (p=0.000 n=20+19) Mutex-16 133ns ± 5% 130ns ± 3% -1.99% (p=0.002 n=20+20) MutexSlack 33.4ns ± 2% 16.2ns ± 0% -51.44% (p=0.000 n=19+20) MutexSlack-4 206ns ± 5% 209ns ± 9% ~ (p=0.154 n=20+20) MutexSlack-16 89.4ns ± 1% 90.9ns ± 2% +1.70% (p=0.000 n=18+17) MutexWork 60.5ns ± 0% 55.3ns ± 1% -8.59% (p=0.000 n=12+20) MutexWork-4 105ns ± 5% 97ns ±11% -7.95% (p=0.000 n=20+20) MutexWork-16 157ns ± 1% 158ns ± 1% +0.66% (p=0.001 n=18+17) MutexWorkSlack 70.2ns ± 5% 55.3ns ± 0% -21.30% (p=0.000 n=19+18) MutexWorkSlack-4 277ns ±13% 260ns ±15% -6.35% (p=0.002 n=20+18) MutexWorkSlack-16 156ns ± 0% 146ns ± 1% -6.40% (p=0.000 n=16+19) MutexNoSpin 966ns ± 0% 976ns ± 1% +0.97% (p=0.000 n=15+17) MutexNoSpin-4 269ns ± 4% 272ns ± 4% +1.15% (p=0.048 n=20+18) MutexNoSpin-16 122ns ± 0% 119ns ± 1% -2.63% (p=0.000 n=19+15) MutexSpin 3.13µs ± 0% 3.12µs ± 0% -0.17% (p=0.000 n=18+18) MutexSpin-4 826ns ± 1% 833ns ± 1% +0.84% (p=0.000 n=19+17) MutexSpin-16 397ns ± 1% 394ns ± 1% -0.78% (p=0.000 n=19+19) Once 5.67ns ± 0% 2.07ns ± 2% -63.43% (p=0.000 n=20+20) Once-4 1.47ns ± 2% 0.54ns ± 3% -63.49% (p=0.000 n=19+20) Once-16 0.58ns ± 0% 0.17ns ± 5% -70.49% (p=0.000 n=17+17) RWMutexUncontended 71.4ns ± 0% 60.3ns ± 1% -15.60% (p=0.000 n=16+20) RWMutexUncontended-4 18.4ns ± 4% 15.3ns ± 1% -17.14% (p=0.000 n=20+20) RWMutexUncontended-16 8.01ns ± 0% 7.69ns ± 0% -3.91% (p=0.000 n=18+15) RWMutexWrite100 24.9ns ± 0% 24.0ns ± 1% -3.57% (p=0.000 n=19+18) RWMutexWrite100-4 46.5ns ± 3% 44.1ns ± 4% -5.09% (p=0.000 n=17+20) RWMutexWrite100-16 68.9ns ± 3% 65.7ns ± 8% -4.65% (p=0.000 n=18+20) RWMutexWrite10 27.1ns ± 0% 25.7ns ± 0% -5.25% (p=0.000 n=17+14) RWMutexWrite10-4 34.8ns ± 1% 33.8ns ± 2% -2.96% (p=0.000 n=20+20) RWMutexWrite10-16 37.5ns ± 2% 36.1ns ± 2% -3.72% (p=0.000 n=20+20) RWMutexWorkWrite100 164ns ± 0% 162ns ± 0% -1.49% (p=0.000 n=12+20) RWMutexWorkWrite100-4 186ns ± 3% 184ns ± 4% ~ (p=0.097 n=20+20) RWMutexWorkWrite100-16 204ns ± 2% 200ns ± 2% -1.58% (p=0.000 n=18+20) RWMutexWorkWrite10 153ns ± 0% 151ns ± 1% -1.21% (p=0.000 n=20+20) RWMutexWorkWrite10-4 179ns ± 1% 176ns ± 2% -1.25% (p=0.000 n=19+20) RWMutexWorkWrite10-16 191ns ± 1% 189ns ± 1% -0.94% (p=0.000 n=15+17) Change-Id: I9269bf2ac42a04c610624f707d3268dcb17390f8 Reviewed-on: https://go-review.googlesource.com/c/go/+/152698 Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Austin Clements <austin@google.com>
diff --git a/src/sync/rwmutex.go b/src/sync/rwmutex.go
@@ -66,21 +66,26 @@ func (rw *RWMutex) RUnlock() {
 		race.Disable()
 	}
 	if r := atomic.AddInt32(&rw.readerCount, -1); r < 0 {
-		if r+1 == 0 || r+1 == -rwmutexMaxReaders {
-			race.Enable()
-			throw("sync: RUnlock of unlocked RWMutex")
-		}
-		// A writer is pending.
-		if atomic.AddInt32(&rw.readerWait, -1) == 0 {
-			// The last reader unblocks the writer.
-			runtime_Semrelease(&rw.writerSem, false, 0)
-		}
+		// Outlined slow-path to allow the fast-path to be inlined
+		rw.rUnlockSlow(r)
 	}
 	if race.Enabled {
 		race.Enable()
 	}
 }
 
+func (rw *RWMutex) rUnlockSlow(r int32) {
+	if r+1 == 0 || r+1 == -rwmutexMaxReaders {
+		race.Enable()
+		throw("sync: RUnlock of unlocked RWMutex")
+	}
+	// A writer is pending.
+	if atomic.AddInt32(&rw.readerWait, -1) == 0 {
+		// The last reader unblocks the writer.
+		runtime_Semrelease(&rw.writerSem, false, 1)
+	}
+}
+
 // Lock locks rw for writing.
 // If the lock is already locked for reading or writing,
 // Lock blocks until the lock is available.
diff --git a/test/inline_sync.go b/test/inline_sync.go
@@ -38,3 +38,16 @@ func small7() { // ERROR "can inline small7"
         // the Do fast path should be inlined
         once.Do(small5) // ERROR "inlining call to sync\.\(\*Once\)\.Do" "&sync\.o\.done escapes to heap"
 }
+
+var rwmutex *sync.RWMutex
+
+func small8() { // ERROR "can inline small8"
+        // the RUnlock fast path should be inlined
+        rwmutex.RUnlock() // ERROR "inlining call to sync\.\(\*RWMutex\)\.RUnlock" "&sync\.rw\.readerCount escapes to heap"
+}
+
+func small9() { // ERROR "can inline small9"
+        // the RLock fast path should be inlined
+        rwmutex.RLock() // ERROR "inlining call to sync\.\(\*RWMutex\)\.RLock" "&sync\.rw\.readerCount escapes to heap" "&sync\.rw\.readerSem escapes to heap"
+}
+
