Skip to content

Commit 0b0cc41

Browse files
josharianjosharian
committed
runtime: refactor walking of bucket overflows
This eliminates a nil check of b while evaluating b.tophash, which is in the inner loop of many hot map functions. It also makes the code a bit clearer. Also remove some gotos in favor of labeled breaks. On non-x86 architectures, this change introduces a pointless reg-reg move, although the cause is well-understood (#21572). Change-Id: Ib7ee58b59ea5463b92e1590c8b8f5c0ef87d410a Reviewed-on: https://go-review.googlesource.com/58372 Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Martin Möhrmann <moehrmann@google.com> Reviewed-by: Keith Randall <khr@golang.org>
1 parent cd5c10f commit 0b0cc41

File tree

2 files changed

+30
-73
lines changed

2 files changed

+30
-73
lines changed

‎src/runtime/hashmap.go‎

Lines changed: 9 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
400400
}
401401
}
402402
top := tophash(hash)
403-
for {
403+
for ; b != nil; b = b.overflow(t) {
404404
for i := uintptr(0); i < bucketCnt; i++ {
405405
if b.tophash[i] != top {
406406
continue
@@ -417,11 +417,8 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
417417
return v
418418
}
419419
}
420-
b = b.overflow(t)
421-
if b == nil {
422-
return unsafe.Pointer(&zeroVal[0])
423-
}
424420
}
421+
return unsafe.Pointer(&zeroVal[0])
425422
}
426423

427424
func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool) {
@@ -455,7 +452,7 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
455452
}
456453
}
457454
top := tophash(hash)
458-
for {
455+
for ; b != nil; b = b.overflow(t) {
459456
for i := uintptr(0); i < bucketCnt; i++ {
460457
if b.tophash[i] != top {
461458
continue
@@ -472,11 +469,8 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
472469
return v, true
473470
}
474471
}
475-
b = b.overflow(t)
476-
if b == nil {
477-
return unsafe.Pointer(&zeroVal[0]), false
478-
}
479472
}
473+
return unsafe.Pointer(&zeroVal[0]), false
480474
}
481475

482476
// returns both key and value. Used by map iterator
@@ -499,7 +493,7 @@ func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe
499493
}
500494
}
501495
top := tophash(hash)
502-
for {
496+
for ; b != nil; b = b.overflow(t) {
503497
for i := uintptr(0); i < bucketCnt; i++ {
504498
if b.tophash[i] != top {
505499
continue
@@ -516,11 +510,8 @@ func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe
516510
return k, v
517511
}
518512
}
519-
b = b.overflow(t)
520-
if b == nil {
521-
return nil, nil
522-
}
523513
}
514+
return nil, nil
524515
}
525516

526517
func mapaccess1_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) unsafe.Pointer {
@@ -681,7 +672,8 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
681672
}
682673
b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
683674
top := tophash(hash)
684-
for {
675+
search:
676+
for ; b != nil; b = b.overflow(t) {
685677
for i := uintptr(0); i < bucketCnt; i++ {
686678
if b.tophash[i] != top {
687679
continue
@@ -711,15 +703,10 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
711703
}
712704
b.tophash[i] = empty
713705
h.count--
714-
goto done
715-
}
716-
b = b.overflow(t)
717-
if b == nil {
718-
goto done
706+
break search
719707
}
720708
}
721709

722-
done:
723710
if h.flags&hashWriting == 0 {
724711
throw("concurrent map writes")
725712
}

‎src/runtime/hashmap_fast.go‎

Lines changed: 21 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,14 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
3939
}
4040
}
4141
}
42-
for {
42+
for ; b != nil; b = b.overflow(t) {
4343
for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
4444
if *(*uint32)(k) == key && b.tophash[i] != empty {
4545
return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
4646
}
4747
}
48-
b = b.overflow(t)
49-
if b == nil {
50-
return unsafe.Pointer(&zeroVal[0])
51-
}
5248
}
49+
return unsafe.Pointer(&zeroVal[0])
5350
}
5451

5552
func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
@@ -82,17 +79,14 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
8279
}
8380
}
8481
}
85-
for {
82+
for ; b != nil; b = b.overflow(t) {
8683
for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
8784
if *(*uint32)(k) == key && b.tophash[i] != empty {
8885
return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)), true
8986
}
9087
}
91-
b = b.overflow(t)
92-
if b == nil {
93-
return unsafe.Pointer(&zeroVal[0]), false
94-
}
9588
}
89+
return unsafe.Pointer(&zeroVal[0]), false
9690
}
9791

9892
func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
@@ -125,17 +119,14 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
125119
}
126120
}
127121
}
128-
for {
122+
for ; b != nil; b = b.overflow(t) {
129123
for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
130124
if *(*uint64)(k) == key && b.tophash[i] != empty {
131125
return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
132126
}
133127
}
134-
b = b.overflow(t)
135-
if b == nil {
136-
return unsafe.Pointer(&zeroVal[0])
137-
}
138128
}
129+
return unsafe.Pointer(&zeroVal[0])
139130
}
140131

141132
func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
@@ -168,17 +159,14 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
168159
}
169160
}
170161
}
171-
for {
162+
for ; b != nil; b = b.overflow(t) {
172163
for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
173164
if *(*uint64)(k) == key && b.tophash[i] != empty {
174165
return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)), true
175166
}
176167
}
177-
b = b.overflow(t)
178-
if b == nil {
179-
return unsafe.Pointer(&zeroVal[0]), false
180-
}
181168
}
169+
return unsafe.Pointer(&zeroVal[0]), false
182170
}
183171

184172
func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
@@ -256,7 +244,7 @@ dohash:
256244
}
257245
}
258246
top := tophash(hash)
259-
for {
247+
for ; b != nil; b = b.overflow(t) {
260248
for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
261249
k := (*stringStruct)(kptr)
262250
if k.len != key.len || b.tophash[i] != top {
@@ -266,11 +254,8 @@ dohash:
266254
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
267255
}
268256
}
269-
b = b.overflow(t)
270-
if b == nil {
271-
return unsafe.Pointer(&zeroVal[0])
272-
}
273257
}
258+
return unsafe.Pointer(&zeroVal[0])
274259
}
275260

276261
func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
@@ -348,7 +333,7 @@ dohash:
348333
}
349334
}
350335
top := tophash(hash)
351-
for {
336+
for ; b != nil; b = b.overflow(t) {
352337
for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
353338
k := (*stringStruct)(kptr)
354339
if k.len != key.len || b.tophash[i] != top {
@@ -358,11 +343,8 @@ dohash:
358343
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)), true
359344
}
360345
}
361-
b = b.overflow(t)
362-
if b == nil {
363-
return unsafe.Pointer(&zeroVal[0]), false
364-
}
365346
}
347+
return unsafe.Pointer(&zeroVal[0]), false
366348
}
367349

368350
func mapassign_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
@@ -647,7 +629,8 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) {
647629
growWork(t, h, bucket)
648630
}
649631
b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
650-
for {
632+
search:
633+
for ; b != nil; b = b.overflow(t) {
651634
for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
652635
if key != *(*uint32)(k) || b.tophash[i] == empty {
653636
continue
@@ -663,15 +646,10 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) {
663646
}
664647
b.tophash[i] = empty
665648
h.count--
666-
goto done
667-
}
668-
b = b.overflow(t)
669-
if b == nil {
670-
goto done
649+
break search
671650
}
672651
}
673652

674-
done:
675653
if h.flags&hashWriting == 0 {
676654
throw("concurrent map writes")
677655
}
@@ -700,7 +678,8 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) {
700678
growWork(t, h, bucket)
701679
}
702680
b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
703-
for {
681+
search:
682+
for ; b != nil; b = b.overflow(t) {
704683
for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
705684
if key != *(*uint64)(k) || b.tophash[i] == empty {
706685
continue
@@ -716,15 +695,10 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) {
716695
}
717696
b.tophash[i] = empty
718697
h.count--
719-
goto done
720-
}
721-
b = b.overflow(t)
722-
if b == nil {
723-
goto done
698+
break search
724699
}
725700
}
726701

727-
done:
728702
if h.flags&hashWriting == 0 {
729703
throw("concurrent map writes")
730704
}
@@ -755,7 +729,8 @@ func mapdelete_faststr(t *maptype, h *hmap, ky string) {
755729
}
756730
b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
757731
top := tophash(hash)
758-
for {
732+
search:
733+
for ; b != nil; b = b.overflow(t) {
759734
for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
760735
k := (*stringStruct)(kptr)
761736
if k.len != key.len || b.tophash[i] != top {
@@ -772,15 +747,10 @@ func mapdelete_faststr(t *maptype, h *hmap, ky string) {
772747
}
773748
b.tophash[i] = empty
774749
h.count--
775-
goto done
776-
}
777-
b = b.overflow(t)
778-
if b == nil {
779-
goto done
750+
break search
780751
}
781752
}
782753

783-
done:
784754
if h.flags&hashWriting == 0 {
785755
throw("concurrent map writes")
786756
}

0 commit comments

Comments
 (0)