4
4
5
5
package sync
6
6
7
+ import (
8
+ "runtime"
9
+ "sync/atomic"
10
+ "unsafe"
11
+ )
12
+
13
+ const (
14
+ cacheLineSize = 128
15
+ poolLocalSize = 2 * cacheLineSize
16
+ poolLocalCap = poolLocalSize / unsafe .Sizeof (* (* interface {})(nil )) - 1
17
+ )
18
+
7
19
// A Pool is a set of temporary objects that may be individually saved
8
20
// and retrieved.
9
21
//
@@ -26,29 +38,52 @@ package sync
26
38
//
27
39
// This is an experimental type and might not be released.
28
40
type Pool struct {
29
- next * Pool // for use by runtime. must be first.
30
- list []interface {} // offset known to runtime
31
- mu Mutex // guards list
41
+ // The following fields are known to runtime.
42
+ next * Pool // for use by runtime
43
+ local * poolLocal // local fixed-size per-P pool, actually an array
44
+ localSize uintptr // size of the local array
45
+ globalOffset uintptr // offset of global
46
+ // The rest is not known to runtime.
32
47
33
48
// New optionally specifies a function to generate
34
49
// a value when Get would otherwise return nil.
35
50
// It may not be changed concurrently with calls to Get.
36
51
New func () interface {}
52
+
53
+ pad [cacheLineSize ]byte
54
+ // Read-mostly date above this point, mutable data follows.
55
+ mu Mutex
56
+ global []interface {} // global fallback pool
37
57
}
38
58
39
- func runtime_registerPool (* Pool )
59
+ // Local per-P Pool appendix.
60
+ type poolLocal struct {
61
+ tail int
62
+ unused int
63
+ buf [poolLocalCap ]interface {}
64
+ }
65
+
66
+ func init () {
67
+ var v poolLocal
68
+ if unsafe .Sizeof (v ) != poolLocalSize {
69
+ panic ("sync: incorrect pool size" )
70
+ }
71
+ }
40
72
41
73
// Put adds x to the pool.
42
74
func (p * Pool ) Put (x interface {}) {
43
75
if x == nil {
44
76
return
45
77
}
46
- p .mu .Lock ()
47
- if p .list == nil {
48
- runtime_registerPool (p )
78
+ l := p .pin ()
79
+ t := l .tail
80
+ if t < int (poolLocalCap ) {
81
+ l .buf [t ] = x
82
+ l .tail = t + 1
83
+ runtime_procUnpin ()
84
+ return
49
85
}
50
- p .list = append (p .list , x )
51
- p .mu .Unlock ()
86
+ p .putSlow (l , x )
52
87
}
53
88
54
89
// Get selects an arbitrary item from the Pool, removes it from the
@@ -60,16 +95,116 @@ func (p *Pool) Put(x interface{}) {
60
95
// If Get would otherwise return nil and p.New is non-nil, Get returns
61
96
// the result of calling p.New.
62
97
func (p * Pool ) Get () interface {} {
98
+ l := p .pin ()
99
+ t := l .tail
100
+ if t > 0 {
101
+ t -= 1
102
+ x := l .buf [t ]
103
+ l .tail = t
104
+ runtime_procUnpin ()
105
+ return x
106
+ }
107
+ return p .getSlow ()
108
+ }
109
+
110
+ func (p * Pool ) putSlow (l * poolLocal , x interface {}) {
111
+ // Grab half of items from local pool and put to global pool.
112
+ // Can not lock the mutex while pinned.
113
+ const N = int (poolLocalCap / 2 + 1 )
114
+ var buf [N ]interface {}
115
+ buf [0 ] = x
116
+ for i := 1 ; i < N ; i ++ {
117
+ l .tail --
118
+ buf [i ] = l .buf [l .tail ]
119
+ }
120
+ runtime_procUnpin ()
121
+
63
122
p .mu .Lock ()
64
- var x interface {}
65
- if n := len (p .list ); n > 0 {
66
- x = p .list [n - 1 ]
67
- p .list [n - 1 ] = nil // Just to be safe
68
- p .list = p .list [:n - 1 ]
123
+ p .global = append (p .global , buf [:]... )
124
+ p .mu .Unlock ()
125
+ }
126
+
127
+ func (p * Pool ) getSlow () (x interface {}) {
128
+ // Grab a batch of items from global pool and put to local pool.
129
+ // Can not lock the mutex while pinned.
130
+ runtime_procUnpin ()
131
+ p .mu .Lock ()
132
+ pid := runtime_procPin ()
133
+ s := p .localSize
134
+ l := p .local
135
+ if uintptr (pid ) < s {
136
+ l = indexLocal (l , pid )
137
+ // Get the item to return.
138
+ last := len (p .global ) - 1
139
+ if last >= 0 {
140
+ x = p .global [last ]
141
+ p .global = p .global [:last ]
142
+ }
143
+ // Try to refill local pool, we may have been rescheduled to another P.
144
+ if last > 0 && l .tail == 0 {
145
+ n := int (poolLocalCap / 2 )
146
+ gl := len (p .global )
147
+ if n > gl {
148
+ n = gl
149
+ }
150
+ copy (l .buf [:], p .global [gl - n :])
151
+ p .global = p .global [:gl - n ]
152
+ l .tail = n
153
+ }
69
154
}
155
+ runtime_procUnpin ()
70
156
p .mu .Unlock ()
157
+
71
158
if x == nil && p .New != nil {
72
159
x = p .New ()
73
160
}
74
- return x
161
+ return
75
162
}
163
+
164
+ // pin pins current goroutine to P, disables preemption and returns poolLocal pool for the P.
165
+ // Caller must call runtime_procUnpin() when done with the pool.
166
+ func (p * Pool ) pin () * poolLocal {
167
+ pid := runtime_procPin ()
168
+ // In pinSlow we store to localSize and then to local, here we load in opposite order.
169
+ // Since we've disabled preemption, GC can not happen in between.
170
+ // Thus here we must observe local at least as large localSize.
171
+ // We can observe a newer/larger local, it is fine (we must observe its zero-initialized-ness).
172
+ s := atomic .LoadUintptr (& p .localSize ) // load-acquire
173
+ l := p .local // load-consume
174
+ if uintptr (pid ) < s {
175
+ return indexLocal (l , pid )
176
+ }
177
+ return p .pinSlow ()
178
+ }
179
+
180
+ func (p * Pool ) pinSlow () * poolLocal {
181
+ // Retry under the mutex.
182
+ runtime_procUnpin ()
183
+ p .mu .Lock ()
184
+ defer p .mu .Unlock ()
185
+ pid := runtime_procPin ()
186
+ s := p .localSize
187
+ l := p .local
188
+ if uintptr (pid ) < s {
189
+ return indexLocal (l , pid )
190
+ }
191
+ if p .local == nil {
192
+ p .globalOffset = unsafe .Offsetof (p .global )
193
+ runtime_registerPool (p )
194
+ }
195
+ // If GOMAXPROCS changes between GCs, we re-allocate the array and lose the old one.
196
+ size := runtime .GOMAXPROCS (0 )
197
+ local := make ([]poolLocal , size )
198
+ atomic .StorePointer ((* unsafe .Pointer )(unsafe .Pointer (& p .local )), unsafe .Pointer (& local [0 ])) // store-release
199
+ atomic .StoreUintptr (& p .localSize , uintptr (size )) // store-release
200
+ return & local [pid ]
201
+ }
202
+
203
+ func indexLocal (l * poolLocal , i int ) * poolLocal {
204
+ return (* poolLocal )(unsafe .Pointer (uintptr (unsafe .Pointer (l )) + unsafe .Sizeof (* l )* uintptr (i ))) // uh...
205
+ }
206
+
207
+ // Implemented in runtime.
208
+ func runtime_registerPool (* Pool )
209
+ func runtime_procPin () int
210
+ func runtime_procUnpin ()
0 commit comments