-
Notifications
You must be signed in to change notification settings - Fork 0
/
atom.xml
452 lines (257 loc) · 378 KB
/
atom.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Cwww3's Blog</title>
<subtitle>Record what you think</subtitle>
<link href="https://cwww3.github.io/atom.xml" rel="self"/>
<link href="https://cwww3.github.io/"/>
<updated>2022-09-15T14:57:26.932Z</updated>
<id>https://cwww3.github.io/</id>
<author>
<name>Cwww3</name>
</author>
<generator uri="https://hexo.io/">Hexo</generator>
<entry>
<title>二分查找</title>
<link href="https://cwww3.github.io/2022/09/15/%E7%AE%97%E6%B3%95/%E4%BA%8C%E5%88%86%E6%9F%A5%E6%89%BE/"/>
<id>https://cwww3.github.io/2022/09/15/%E7%AE%97%E6%B3%95/%E4%BA%8C%E5%88%86%E6%9F%A5%E6%89%BE/</id>
<published>2022-09-15T14:48:00.000Z</published>
<updated>2022-09-15T14:57:26.932Z</updated>
<content type="html"><![CDATA[<h3 id="二分查找"><a href="#二分查找" class="headerlink" title="二分查找"></a>二分查找</h3><h4 id="等于目标元素的索引"><a href="#等于目标元素的索引" class="headerlink" title="等于目标元素的索引"></a>等于目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getTarget</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> {</span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">2</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] == target {</span><br><span class="line"><span class="keyword">return</span> mid</span><br><span class="line">} <span class="keyword">else</span> <span class="keyword">if</span> nums[mid] > target {</span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> <span class="number">-1</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><span id="more"></span><h4 id="第一个大于等于目标元素的索引"><a href="#第一个大于等于目标元素的索引" class="headerlink" title="第一个大于等于目标元素的索引"></a>第一个大于等于目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getFirstTarget</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> {</span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">2</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] >= target {</span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> left</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="最后一个小于等于目标元素的索引"><a href="#最后一个小于等于目标元素的索引" class="headerlink" title="最后一个小于等于目标元素的索引"></a>最后一个小于等于目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getLastTarget</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> {</span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">2</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] <= target {</span><br><span class="line"> left = mid + <span class="number">1</span></span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> right</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="第一个大于目标元素的索引"><a href="#第一个大于目标元素的索引" class="headerlink" title="第一个大于目标元素的索引"></a>第一个大于目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getFirstLargerTarget</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> {</span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">2</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] > target {</span><br><span class="line"><span class="keyword">if</span> mid == <span class="number">0</span> || nums[mid<span class="number">-1</span>] <= target {</span><br><span class="line"><span class="comment">// mid == 0 所有值都比target大</span></span><br><span class="line"><span class="comment">// nums[mid-1] <= target 说明mid是第一个比目标值大的索引</span></span><br><span class="line"><span class="keyword">return</span> mid</span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">} <span class="keyword">else</span> <span class="keyword">if</span> nums[mid] <= target {</span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="comment">//所有元素都小于目标元素</span></span><br><span class="line"><span class="keyword">return</span> <span class="number">-1</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="最后一个小于目标元素的索引"><a href="#最后一个小于目标元素的索引" class="headerlink" title="最后一个小于目标元素的索引"></a>最后一个小于目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getLastSmall</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> {</span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">2</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] >= target {</span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">} <span class="keyword">else</span> nums[mid] < target {</span><br><span class="line"><span class="keyword">if</span> mid == <span class="built_in">len</span>(nums)<span class="number">-1</span> || nums[mid+<span class="number">1</span>] >= target {</span><br><span class="line"><span class="comment">// mid == len(nums)-1 所有元素都小于目标元素</span></span><br><span class="line"><span class="comment">// nums[mid+1] >= target mid是最后一个小于目标元素的索引</span></span><br><span class="line"><span class="keyword">return</span> mid</span><br><span class="line">}</span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="comment">//所有元素都大于目标元素</span></span><br><span class="line"><span class="keyword">return</span> <span class="number">-1</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="在旋转有序数组中找目标元素的索引"><a href="#在旋转有序数组中找目标元素的索引" class="headerlink" title="在旋转有序数组中找目标元素的索引"></a>在旋转有序数组中找目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// [5,6,7,1,2,3]</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getTargetInLoop</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> {</span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">1</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] == target {</span><br><span class="line"><span class="keyword">return</span> mid</span><br><span class="line">}</span><br><span class="line"><span class="keyword">if</span> nums[left] <= nums[mid] { <span class="comment">// left和right相差1时，mid=left</span></span><br><span class="line"><span class="comment">// 说明 left和mid在同一个有序数组里</span></span><br><span class="line"><span class="keyword">if</span> target >= nums[left] && target < nums[mid] {</span><br><span class="line"><span class="comment">// 说明 target 在 left和mid中间 需要移动right</span></span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">}<span class="keyword">else</span> {</span><br><span class="line"><span class="comment">// target 和 (left/mid)不在同一个有序数组</span></span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line"><span class="comment">// 说明 mid和right在同一个有序数组里</span></span><br><span class="line"><span class="keyword">if</span> nums[mid] < target && nums[right] >= target {</span><br><span class="line"><span class="comment">// 说明 target 在mid和right中间 需要移动left</span></span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}<span class="keyword">else</span>{</span><br><span class="line"><span class="comment">// target 和 (mid/right)不在同一个有序数组</span></span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> <span class="number">-1</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="二维数组中查找目标元素的索引"><a href="#二维数组中查找目标元素的索引" class="headerlink" title="二维数组中查找目标元素的索引"></a>二维数组中查找目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getTargetInTwoDimensional</span><span class="params">(nums [][]<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">bool</span></span> {</span><br><span class="line">line := <span class="built_in">len</span>(nums)</span><br><span class="line"> <span class="keyword">if</span> line == <span class="number">0</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">false</span></span><br><span class="line"> }</span><br><span class="line">col := <span class="built_in">len</span>(nums[<span class="number">0</span>])</span><br><span class="line"></span><br><span class="line">left := <span class="number">0</span></span><br><span class="line">right := line*col - <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> left <= right {</span><br><span class="line">mid := left + (right-left)>><span class="number">2</span></span><br><span class="line">x := mid / col</span><br><span class="line">y := mid % col</span><br><span class="line"><span class="keyword">if</span> nums[x][y] == target {</span><br><span class="line"><span class="keyword">return</span> <span class="literal">true</span></span><br><span class="line">} <span class="keyword">else</span> <span class="keyword">if</span> nums[x][y] > target {</span><br><span class="line">right = mid - <span class="number">1</span></span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line">left = mid + <span class="number">1</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> <span class="literal">false</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><h3 id="二分查找"><a href="#二分查找" class="headerlink" title="二分查找"></a>二分查找</h3><h4 id="等于目标元素的索引"><a href="#等于目标元素的索引" class="headerlink" title="等于目标元素的索引"></a>等于目标元素的索引</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">getTarget</span><span class="params">(nums []<span class="keyword">int</span>, target <span class="keyword">int</span>)</span> <span class="title">int</span></span> &#123;</span><br><span class="line"> left := <span class="number">0</span></span><br><span class="line"> right := <span class="built_in">len</span>(nums) - <span class="number">1</span></span><br><span class="line"> <span class="keyword">for</span> left &lt;= right &#123;</span><br><span class="line"> mid := left + (right-left)&gt;&gt;<span class="number">2</span></span><br><span class="line"> <span class="keyword">if</span> nums[mid] == target &#123;</span><br><span class="line"> <span class="keyword">return</span> mid</span><br><span class="line"> &#125; <span class="keyword">else</span> <span class="keyword">if</span> nums[mid] &gt; target &#123;</span><br><span class="line"> right = mid - <span class="number">1</span></span><br><span class="line"> &#125; <span class="keyword">else</span> &#123;</span><br><span class="line"> left = mid + <span class="number">1</span></span><br><span class="line"> &#125;</span><br><span class="line"> &#125;</span><br><span class="line"> <span class="keyword">return</span> <span class="number">-1</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></summary>
<category term="算法" scheme="https://cwww3.github.io/tags/%E7%AE%97%E6%B3%95/"/>
</entry>
<entry>
<title>DNS</title>
<link href="https://cwww3.github.io/2022/09/10/system-design/dns/"/>
<id>https://cwww3.github.io/2022/09/10/system-design/dns/</id>
<published>2022-09-10T07:54:25.000Z</published>
<updated>2022-09-10T07:55:31.404Z</updated>
<content type="html"><![CDATA[<p>域名系统 DNS(domain name system)是因特网使用的命名系统，用于把便于人们记忆的机器名字(域名)转换成 ip 地址。</p><h3 id="域名分级"><a href="#域名分级" class="headerlink" title="域名分级"></a>域名分级</h3><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220910153820132.png" alt="image-20220910153820132"></p><span id="more"></span><h3 id="域名服务器分类"><a href="#域名服务器分类" class="headerlink" title="域名服务器分类"></a>域名服务器分类</h3><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220910153902956.png" alt="image-20220910153902956"></p><h4 id="根服务器"><a href="#根服务器" class="headerlink" title="根服务器"></a>根服务器</h4><p>根域名服务器是最高层次的域名服务器，<strong>所有的根域名服务器都知道所有顶级域名服务器的域名和 ip</strong>。根域名服务器一版情况下不会把待查询的域名直接转换 IP，而是告诉本地域名服务器下一步应该找哪一个顶级域名服务器进行查询。</p><h4 id="顶级域名服务器"><a href="#顶级域名服务器" class="headerlink" title="顶级域名服务器"></a>顶级域名服务器</h4><p>顶级域名服务器管理在该顶级域名服务器注册的所有二级域名，但受到 DNS 查询就会有相应应答。（可能是给出最后的结果或下一步一应当找的域名服务器 ip）</p><h4 id="权限域名服务器"><a href="#权限域名服务器" class="headerlink" title="权限域名服务器"></a>权限域名服务器</h4><p>可以理解为二级域名下负责一个区的域名服务器、如 abc.com 和 y.abc.com 应各设有一个权限域名服务器。<br>如果访问量较小，整个 abc.com 仅一个区。下面的 y.abc.com 和 a.abc.com 应该是公用的同一个权限域名服务器。</p><h4 id="本地域名服务器"><a href="#本地域名服务器" class="headerlink" title="本地域名服务器"></a>本地域名服务器</h4><p><strong>本地域名服务器并不属于域名服务器图中的层级结构，但对域名系统非常重要</strong>。但一台主机发出 DNS 请求时就是发给本地域名服务器。每一个因特网服务提供 ISP(电信联通移动)，或一个大学都可以拥有一个本地域名服务器。这种服务器有时也被称为默认域名服务器。本地域名服务器一般离用户较近，一般不超过几个路由的距离。如果要查询的 IP 同属一个本地 ISP 时即可直接返回结果地址 ip。</p><h3 id="DNS-记录类型"><a href="#DNS-记录类型" class="headerlink" title="DNS 记录类型"></a>DNS 记录类型</h3><p><strong>A 记录</strong>：地址记录（Address），返回域名指向的 IP 地址。</p><p><strong>AAAA 记录</strong>: 返回域名指向的 IPv6 地址。</p><p><strong>NS 记录</strong>：域名服务器记录（Name Server），返回保存下一级域名信息的服务器地址。该记录只能设置为域名，不能设置为 IP 地址。</p><p><strong>CNAME 记录</strong>：规范名称记录（Canonical Name），返回另一个域名,相当于别名，即当前查询的域名是另一个域名的跳转。</p><h3 id="域名解析过程"><a href="#域名解析过程" class="headerlink" title="域名解析过程"></a>域名解析过程</h3><p>DNS 解析流程：主机与本地域名服务器递归查询，本地域名服务器迭代查询(一般情况)</p><h4 id="递归模式"><a href="#递归模式" class="headerlink" title="递归模式"></a>递归模式</h4><p>如果主机所询问的本地域名服务器不知道被查询的域名的 IP 地址，那么本地域名服务器就以 DNS 客户的身份，向其它根域名服务器继续发出查询请求报文(即替主机继续查询)，而不是让主机自己进行下一步查询。</p><h4 id="迭代模式"><a href="#迭代模式" class="headerlink" title="迭代模式"></a>迭代模式</h4><p>当根域名服务器收到本地域名服务器发出的迭代查询请求报文时，要么给出所要查询的 IP 地址，要么告诉本地服务器：应当向哪一个域名服务器进行查询。然后让本地服务器进行后续的查询。顶级域名服务器在收到本地域名服务器的查询请求后，要么给出所要查询的 IP 地址，要么告诉本地服务器下一步应当向哪一个权限域名服务器进行查询。最后，知道了所要解析的 IP 地址，然后把这个结果返回给发起查询的主机。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/6bb796a3045e409aabb0f89ad40d3fad%7Etplv-k3u1fbpfcp-zoom-in-crop-mark%3A3024%3A0%3A0%3A0.awebp" alt="DNS.png"></p><h3 id="引用"><a href="#引用" class="headerlink" title="引用"></a>引用</h3><p><a href="https://blog.csdn.net/m0_37263637/article/details/85157611">https://blog.csdn.net/m0_37263637/article/details/85157611</a></p><p><a href="https://juejin.cn/post/6990344840181940261">https://juejin.cn/post/6990344840181940261</a></p>]]></content>
<summary type="html"><p>域名系统 DNS(domain name system)是因特网使用的命名系统，用于把便于人们记忆的机器名字(域名)转换成 ip 地址。</p>
<h3 id="域名分级"><a href="#域名分级" class="headerlink" title="域名分级"></a>域名分级</h3><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220910153820132.png" alt="image-20220910153820132"></p></summary>
<category term="System Design" scheme="https://cwww3.github.io/tags/System-Design/"/>
<category term="DNS" scheme="https://cwww3.github.io/tags/DNS/"/>
</entry>
<entry>
<title>负载均衡</title>
<link href="https://cwww3.github.io/2022/09/10/system-design/loadbalancer/"/>
<id>https://cwww3.github.io/2022/09/10/system-design/loadbalancer/</id>
<published>2022-09-10T07:27:33.000Z</published>
<updated>2022-09-10T07:28:17.659Z</updated>
<content type="html"><![CDATA[<h3 id="负载均衡"><a href="#负载均衡" class="headerlink" title="负载均衡"></a>负载均衡</h3><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/2ff1c9ff90f54148a3196e99524ca509.png" alt="img"></p><span id="more"></span><h4 id="四层负载均衡"><a href="#四层负载均衡" class="headerlink" title="四层负载均衡"></a>四层负载均衡</h4><p>四层负载均衡工作在 OSI 七层模型的第四层（传输层），指的是负载均衡设备<strong>通过报文中的目标 IP 地址、端口和负载均衡算法，选择到达的目标内部服务器</strong>，<strong>四层负载均衡对数据包只起一个数据转发的作用，无法修改或判断所请求资源的具体类型，也不会干预客户端与服务器之间应用层的通信（如三次握手等）</strong>。但在某些部署情况下，为保证服务器回包可以正确返回给负载均衡设备，在转发报文的同时可能会对报文原来的源地址进行修改。</p><p>四层负载均衡单纯的提供了终端到终端的可靠连接，并将请求转发至后端，<strong>连接至始至终都是同一个</strong>。LVS 就是很典型的四层负载均衡。</p><h4 id="七层负载均衡："><a href="#七层负载均衡：" class="headerlink" title="七层负载均衡："></a>七层负载均衡：</h4><p> 七层负载均衡工作在 OSI 模型的第七层（应用层），指的是负载均衡设备<strong>通过请求报文中的应用层信息(如 URL、HTTP 头部、资源类型等信息)和负载均衡算法，选择到达的目标内部服务器</strong>。七层负载均衡的功能更加丰富灵活，另外七层负载均衡两端（面向用户端和服务器端）的连接都是独立的，在一定程度上也提升了后端系统的安全性，四层负载均衡与服务器直接建立起 TCP 连接，很容易遭受 SYN Flood 攻击。SYN Flood 是一种广为人知的 DDoS（分布式拒绝服务攻击）的方式之一，这是一种利用 TCP 协议缺陷，发送大量伪造的 TCP 连接请求，从而使得被攻击方资源耗尽的攻击方式。从技术实现原理上可以看出，四层负载均衡很容易将垃圾流量转发至后台服务器，而七层设备则可以过滤这些恶意并清洗这些流量，但要求设备本身具备很强的抗 DDOS 流量的能力。比如常见 Nginx 就是运行在七层的负载均衡软件</p><h4 id="引用"><a href="#引用" class="headerlink" title="引用"></a>引用</h4><p><a href="https://blog.csdn.net/a745233700/article/details/122445229">https://blog.csdn.net/a745233700/article/details/122445229</a></p>]]></content>
<summary type="html"><h3 id="负载均衡"><a href="#负载均衡" class="headerlink" title="负载均衡"></a>负载均衡</h3><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/2ff1c9ff90f54148a3196e99524ca509.png" alt="img"></p></summary>
<category term="System Design" scheme="https://cwww3.github.io/tags/System-Design/"/>
<category term="负载均衡" scheme="https://cwww3.github.io/tags/%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1/"/>
</entry>
<entry>
<title>CDN</title>
<link href="https://cwww3.github.io/2022/09/09/system-design/cdn/"/>
<id>https://cwww3.github.io/2022/09/09/system-design/cdn/</id>
<published>2022-09-08T16:22:37.000Z</published>
<updated>2022-09-08T16:26:59.475Z</updated>
<content type="html"><![CDATA[<p>CDN 是构建在<strong>现有网络基础之上的智能虚拟网络</strong>，依靠部署在各地的<strong>边缘服务器</strong>，通过中心平台的负载均衡、内容分发、调度等功能模块，<strong>使用户就近获取所需内容</strong>，降低网络拥塞，<strong>提高用户访问响应速度和命中率</strong>。</p><span id="more"></span><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/ec8346c69f1e4f33af93c8cc6fd991e1~tplv-k3u1fbpfcp-zoom-in-crop-mark%3A4536%3A0%3A0%3A0.awebp" alt="img"></p><p>这幅图展示了互联网通信领域中常说的”<strong>三公里</strong>“：</p><ul><li><strong>第一公里</strong><br>网站服务器接入互联网公网的链路，这里的带宽也决定了网站的负载能力，也称为网站的接入带宽。</li><li><strong>中间一公里</strong><br>中间一公里主要是接入网、城域网、骨干网组成的链路实体，其中会涉及多家运营商，也就出现了运营商之间互联互通的数据交换问题。</li><li><strong>最后一公里</strong><br>这是用户接入互联网获取信息的最后环节</li></ul><p>运营商之间数据的互联互通问题，比如 A 市联通要访问 A 市电信的数据资源，按照互联互通的规则限制，不同运营商的数据要<strong>在指定的交换中心进行数据交换</strong>，假如交换中心位于较远的 B 市，本来两个运营商是同一个城市的，但由于运营商的网络差异需要到几百公里之外的交换中心所在的城市进行数据交换，实现资源的访问</p><h4 id="传统-DNS"><a href="#传统-DNS" class="headerlink" title="传统 DNS"></a>传统 DNS</h4><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/35328dd8b2d743069aa538521d0089c2~tplv-k3u1fbpfcp-zoom-in-crop-mark%3A4536%3A0%3A0%3A0.awebp" alt="img"></p><h4 id="有-CDN-参与的-DNS-调度过程"><a href="#有-CDN-参与的-DNS-调度过程" class="headerlink" title="有 CDN 参与的 DNS 调度过程"></a>有 CDN 参与的 DNS 调度过程</h4><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/539a70a65ed44530af51a4d1b8d31df9~tplv-k3u1fbpfcp-zoom-in-crop-mark%3A4536%3A0%3A0%3A0.awebp" alt="img"></p><p>CDN 服务商提供的<strong>专用 DNS 调度服务器</strong>(TenCent DNS Server)根据 CDN 系统内部节点的位置、负载情况、资源分配等因素选出最优的 CDN 资源节点 IP 地址返回给用户</p><h4 id="静态加速"><a href="#静态加速" class="headerlink" title="静态加速"></a>静态加速</h4><p>如果每个用户访问得到的资源一样，就像电视台播放节目，大家看到的都一样，并非个性化的结果，这类资源就可以称为静态资源。 比如网站的图片、视频、软件安装包、各类下载资源文件等。</p><p>这些资源变化很小，因此非常使用 CDN 加速，对改善网站性能效果明显。</p><h4 id="动态加速"><a href="#动态加速" class="headerlink" title="动态加速"></a>动态加速</h4><p>区别于静态资源，动态资源则更倾向于接口、个性化内容，用户每次请求得到的结果可能不同，这些资源并不适合 CDN 场景，如果强行使用会带来数据更新缓慢和不一致问题，但是动态资源有其特有的加速方法。</p><p>动态资源就意味着回源站进行数据请求，这其中就涉及到<strong>最优回源路径的选择</strong>，让路更好走，数据获取更快捷，实现动态资源的加速。</p><h4 id="引用"><a href="#引用" class="headerlink" title="引用"></a>引用</h4><p><a href="https://juejin.cn/post/7064952956201730062#heading-14">https://juejin.cn/post/7064952956201730062#heading-14</a></p>]]></content>
<summary type="html"><p>CDN 是构建在<strong>现有网络基础之上的智能虚拟网络</strong>，依靠部署在各地的<strong>边缘服务器</strong>，通过中心平台的负载均衡、内容分发、调度等功能模块，<strong>使用户就近获取所需内容</strong>，降低网络拥塞，<strong>提高用户访问响应速度和命中率</strong>。</p></summary>
<category term="System Design" scheme="https://cwww3.github.io/tags/System-Design/"/>
<category term="CDN" scheme="https://cwww3.github.io/tags/CDN/"/>
</entry>
<entry>
<title>GO-GC</title>
<link href="https://cwww3.github.io/2022/09/08/GC/"/>
<id>https://cwww3.github.io/2022/09/08/GC/</id>
<published>2022-09-07T16:48:38.000Z</published>
<updated>2022-09-07T17:01:07.871Z</updated>
<content type="html"><![CDATA[<h3 id="GO-1-3-标记清除"><a href="#GO-1-3-标记清除" class="headerlink" title="GO 1.3 标记清除"></a>GO 1.3 标记清除</h3><pre class="mermaid">flowchart LR id1(start STW) --> Mark --> Sweep --> id2(stop SWT)</pre><pre class="mermaid">flowchart LR id1(start STW) --> Mark --> id2(stop SWT) --> Sweep</pre><p>将清除操作置后，缩短了 SWT 时间</p><h3 id="Go-1-5-三色标记法"><a href="#Go-1-5-三色标记法" class="headerlink" title="Go 1.5 三色标记法"></a>Go 1.5 三色标记法</h3><pre class="mermaid">flowchart LR id1(所有对象标记白色) --> id2(根对象标记灰色) --> id3(灰色对象标记为黑色, 灰色对象的子节点标记为灰色) id3 -->|还有灰色对象| id3 id3 -->|没有灰色对象| id4(清除剩余的白色对象)</pre><p>三色标记需要借助 STW，如果没有 STW，在标记过程中可能会出现 ：</p><ul><li>灰色对象删除了子对象的引用(灰色丢失了白色)</li><li>黑色对象引用了被删除的子对象(白色对象被挂在黑色下)</li></ul><p>当上述两个条件同时满足，虽然黑色对象引用着白色对象，但是白色对象最终还是会被清除。</p><span id="more"></span><h3 id="屏障机制"><a href="#屏障机制" class="headerlink" title="屏障机制"></a>屏障机制</h3><p>该技术能够保证内存操作的顺序性，在内存屏障前执行的操作一定会先于内存屏障后执行的操作。</p><p>想要在并发或者增量的标记算法中保证正确性，我们需要达成以下两种三色不变性（Tri-color invariant）中的一种：</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220907093013770.png" alt="image-20220907093013770"></p><h4 id="三色不变性"><a href="#三色不变性" class="headerlink" title="三色不变性"></a>三色不变性</h4><ul><li>强三色不变式</li></ul><p>黑色对象不能直接引用白色对象</p><ul><li>弱三色不变式</li></ul><p>被黑色对象引用的白色对象，它的上游必须有灰色对象</p><p>垃圾收集中的屏障技术更像是一个<strong>钩子方法</strong>，它是在用户程序读取对象、创建新对象以及更新对象指针时执行的一段代码，根据操作类型的不同，我们可以将它们分成<strong>读屏障</strong>（Read barrier）和<strong>写屏障</strong>（Write barrier）两种，因为读屏障需要在读操作中加入代码片段，对用户程序的性能影响很大，所以编程语言往往都会采用写屏障保证三色不变性。</p><h4 id="插入写屏障"><a href="#插入写屏障" class="headerlink" title="插入写屏障"></a>插入写屏障</h4><p>Dijkstra 在 1978 年提出了插入写屏障，通过如下所示的写屏障，用户程序和垃圾收集器可以在交替工作的情况下保证程序执行的正确性：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">writePointer(slot, ptr):</span><br><span class="line"> shade(ptr)</span><br><span class="line"> *slot = ptr</span><br></pre></td></tr></table></figure><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220907144235864.png" alt="image-20220907144235864" style="zoom:33%;" /><p>上述插入写屏障的伪代码非常好理解，每当执行类似 <code>*slot = ptr</code> 的表达式时，我们会执行上述写屏障通过 <code>shade</code> 函数尝试改变指针的颜色。如果 <code>ptr</code> 指针是白色的，那么该函数会将该对象设置成灰色，其他情况则保持不变。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/2020-03-16-15843705141840-dijkstra-insert-write-barrier.png" alt="dijkstra-insert-write-barrier"></p><p>Dijkstra 的插入写屏障是一种<strong>相对保守</strong>的屏障技术，它会将<strong>有存活可能的对象都标记成灰色</strong>以满足<strong>强三色不变性</strong>。实际上不再存活的 B 对象最后没有被回收，只有在下一个循环才会被回收。</p><p><strong>缺点</strong></p><p>因为栈上的对象在垃圾收集中也会被认为是根对象，所以为了保证内存的安全，Dijkstra 必须<strong>为栈上的对象增加写屏障</strong>或者<strong>在标记阶段完成重新对栈上的对象进行扫描</strong>。</p><p>这两种方法各有各的缺点，前者会大幅度增加写入指针的额外开销，后者重新扫描栈对象时需要暂停程序，垃圾收集算法的设计者需要在这两者之间做出权衡。</p><p>Go 选不在栈上开启写屏障，则会导致栈上的黑色对象可能指向白色对象(无法将后来被栈上对象所引用的白色对象变灰)，所以在标记结束后还需要对栈进行 re-scan。</p><h4 id="删除写屏障"><a href="#删除写屏障" class="headerlink" title="删除写屏障"></a>删除写屏障</h4><p>Yuasa 在 1990 年提出了删除写屏障，因为一旦该写屏障开始工作，它会保证开启写屏障时堆上所有对象的可达，所以也被称作快照垃圾收集（Snapshot GC）。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">writePointer(slot, ptr)</span><br><span class="line"> shade(*slot)</span><br><span class="line"> *slot = ptr</span><br></pre></td></tr></table></figure><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220907144106522.png" alt="image-20220907144106522" style="zoom:33%;" /><p>上述代码会在老对象的引用被删除时，将白色的老对象涂成灰色，这样删除写屏障就可以保证弱三色不变性，老对象引用的下游对象一定可以被灰色对象引用。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/2021-01-02-16095599123266-yuasa-delete-write-barrier.png" alt="yuasa-delete-write-barrier"></p><p><strong>缺点</strong></p><p>如果不在栈上开启写屏障，则会导致堆上的黑色对象可能指向白色对象(无法在原先被栈上对象所引用的白色对象被删除时将其变灰)</p><h3 id="Go-1-8-混合写屏障"><a href="#Go-1-8-混合写屏障" class="headerlink" title="Go 1.8 混合写屏障"></a>Go 1.8 混合写屏障</h3><p>在 Go 语言 v1.7 版本之前，运行时会使用 Dijkstra 插入写屏障保证强三色不变性，但是<strong>运行时并没有在所有的垃圾收集根对象上开启插入写屏障</strong>。因为应用程序可能包含成百上千的 Goroutine，而垃圾收集的<strong>根对象一般包括全局变量和栈对象</strong>，如果运行时需要在几百个 Goroutine 的栈上都开启写屏障，会带来巨大的额外开销，所以 Go 团队在实现上选择了不在栈上开启写屏障 ，在标记阶段完成时<strong>暂停程序、将所有栈对象标记为灰色并重新扫描</strong>，在活跃 Goroutine 非常多的程序中，重新扫描的过程需要占用 10 ~ 100ms 的时间。</p><p>Go 语言在 v1.8 组合 Dijkstra 插入写屏障和 Yuasa 删除写屏障构成了如下所示的混合写屏障，该写屏障会<strong>将被覆盖的对象标记成灰色并在当前栈没有扫描时将新对象也标记成灰色</strong>：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">writePointer(slot, ptr):</span><br><span class="line"> shade(*slot)</span><br><span class="line"> shade(ptr)</span><br><span class="line"> *slot = ptr</span><br></pre></td></tr></table></figure><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220907144805355.png" alt="image-20220907144805355" style="zoom:33%;" /><p>为了<strong>移除栈的重扫描过程</strong>，除了引入混合写屏障之外，在垃圾收集的标记阶段，我们还需要<strong>将创建的所有新对象都标记成黑色</strong>，防止新分配的栈内存和堆内存中的对象被错误地回收，因为栈内存在标记阶段最终都会变为黑色，所以不再需要重新扫描栈空间。</p><h3 id="增量和并发"><a href="#增量和并发" class="headerlink" title="增量和并发"></a>增量和并发</h3><p>今天的计算机往往都是多核的处理器，垃圾收集器一旦开始执行就会浪费大量的计算资源，为了减少应用程序暂停的最长时间和垃圾收集的总暂停时间，我们会使用下面的策略优化现代的垃圾收集器：</p><ul><li>增量垃圾收集 — 增量地标记和清除垃圾，降低应用程序暂停的最长时间；</li><li>并发垃圾收集 — 利用多核的计算资源，在用户程序执行时并发标记和清除垃圾；</li></ul><p>因为增量和并发两种方式都可以与用户程序交替运行，所以我们需要<strong>使用屏障技术</strong>保证垃圾收集的正确性；与此同时，应用程序也不能等到内存溢出时触发垃圾收集，因为当内存不足时，应用程序已经无法分配内存，这与直接暂停程序没有什么区别，增量和并发的垃圾收集需要提前触发并在内存不足前完成整个循环，避免程序的长时间暂停。</p><h4 id="增量"><a href="#增量" class="headerlink" title="增量"></a>增量</h4><p>增量式的垃圾收集需要与三色标记法一起使用，为了保证垃圾收集的正确性，我们需要在垃圾收集开始前打开写屏障，这样用户程序修改内存都会先经过写屏障的处理，保证了堆内存中对象关系的强三色不变性或者弱三色不变性。<strong>虽然增量式的垃圾收集能够减少最大的程序暂停时间，但是增量式收集也会增加一次 GC 循环的总时间</strong>，在垃圾收集期间，因为写屏障的影响用户程序也需要承担额外的计算开销，所以增量式的垃圾收集也不是只带来好处的，但是总体来说还是利大于弊。</p><h4 id="并发"><a href="#并发" class="headerlink" title="并发"></a>并发</h4><p>并发的垃圾收集不仅能够减少程序的最长暂停时间，还能减少整个垃圾收集阶段的时间，通过开启读写屏障、<strong>利用多核优势与用户程序并行执行</strong>，并发垃圾收集器确实能够减少垃圾收集对应用程序的影响：</p><p>虽然并发收集器能够与用户程序一起运行，但是并不是所有阶段都可以与用户程序一起运行，部分阶段还是需要暂停用户程序的，不过与传统的算法相比，并发的垃圾收集可以将能够并发执行的工作尽量并发执行；当然，因为读写屏障的引入，并发的垃圾收集器也一定会带来额外开销，不仅会增加垃圾收集的总时间，还会影响用户程序，这是我们在设计垃圾收集策略时必须要注意的。</p><h3 id="GC-触发时机"><a href="#GC-触发时机" class="headerlink" title="GC 触发时机"></a>GC 触发时机</h3><ul><li>内存分配时，目前触发 GC 的条件使用的是从 Go 1.5 时提出的<strong>调步（Pacing）算法</strong>， 调步算法是优化并发执行时 GC 的步调，换句话说就是解决什么时候应该触发下一次 GC 的这个问题。</li><li>用户代码触发</li><li>后台定时触发</li></ul><h3 id="分代"><a href="#分代" class="headerlink" title="分代"></a>分代</h3><p>分代假设并不适用于 Go 的运行栈机制，年轻代对象在栈上就已经死亡，扫描本就该回收的执行栈并没有为由于分代假设带来明显的性能提升，也成为了这一方案最终没有被采用的主要原因。</p><h3 id="小节"><a href="#小节" class="headerlink" title="小节"></a>小节</h3><p>并发回收的屏障技术归根结底就是在利用内存写屏障来保证强三色不变性和弱三色不变性。 早期的 Go 团队实践中选择了从提出较早的 Dijkstra 插入屏障出发， 不可避免的在为了保证强三色不变性的情况下，需要对栈进行重扫。 而在后期的实践中，Go 团队提出了将 Dijkstra 和 Yuasa 屏障结合的混合屏障， 将强三色不变性进行了弱化，从而消除了对栈的重新扫描这一硬性要求，使得在未来实现全面并发 GC 成为可能。</p><p>GoGC 在延迟和吞吐量之间选择了降低延迟(减少了 STW 的时间)</p><h3 id="引用"><a href="#引用" class="headerlink" title="引用"></a>引用</h3><p><a href="https://golang.design/under-the-hood/zh-cn/part2runtime/ch08gc/">GO 语言原本</a></p><p><a href="http://static.kancloud.cn/aceld/golang/1958308">三色标记</a></p><p><a href="https://blog.plan99.net/modern-garbage-collection-911ef4f8bd8e">GC</a></p><p><a href="https://go.dev/blog/go15gc">blog</a></p><p><a href="https://making.pusher.com/golangs-real-time-gc-in-theory-and-practice/index.html">并发标记动画</a></p><p>[如何标记](</p>]]></content>
<summary type="html"><h3 id="GO-1-3-标记清除"><a href="#GO-1-3-标记清除" class="headerlink" title="GO 1.3 标记清除"></a>GO 1.3 标记清除</h3><pre class="mermaid">flowchart LR
id1(start STW) --> Mark --> Sweep --> id2(stop SWT)</pre>
<pre class="mermaid">
flowchart LR
id1(start STW) --> Mark --> id2(stop SWT) --> Sweep</pre>
<p>将清除操作置后，缩短了 SWT 时间</p>
<h3 id="Go-1-5-三色标记法"><a href="#Go-1-5-三色标记法" class="headerlink" title="Go 1.5 三色标记法"></a>Go 1.5 三色标记法</h3><pre class="mermaid">flowchart LR
id1(所有对象标记白色) --> id2(根对象标记灰色) --> id3(灰色对象标记为黑色, 灰色对象的子节点标记为灰色)
id3 -->|还有灰色对象| id3
id3 -->|没有灰色对象| id4(清除剩余的白色对象)</pre>
<p>三色标记需要借助 STW，如果没有 STW，在标记过程中可能会出现 ：</p>
<ul>
<li>灰色对象删除了子对象的引用(灰色丢失了白色)</li>
<li>黑色对象引用了被删除的子对象(白色对象被挂在黑色下)</li>
</ul>
<p>当上述两个条件同时满足，虽然黑色对象引用着白色对象，但是白色对象最终还是会被清除。</p></summary>
<category term="go" scheme="https://cwww3.github.io/tags/go/"/>
<category term="GC" scheme="https://cwww3.github.io/tags/GC/"/>
</entry>
<entry>
<title>go-memery</title>
<link href="https://cwww3.github.io/2022/09/04/go-memory/"/>
<id>https://cwww3.github.io/2022/09/04/go-memory/</id>
<published>2022-09-04T12:10:06.000Z</published>
<updated>2022-09-04T12:12:03.187Z</updated>
<content type="html"><![CDATA[<h2 id="内存管理"><a href="#内存管理" class="headerlink" title="内存管理"></a>内存管理</h2><h3 id="TCMalloc"><a href="#TCMalloc" class="headerlink" title="TCMalloc"></a>TCMalloc</h3><p>TCMalloc 为每个 Thread 预分配一块缓存，每个 Thread 在申请内存时首先会先从这个缓存区 ThreadCache 申请，且所有 ThreadCache 缓存区还共享一个叫 CentralCache 的中心缓存。这里假设目前 Golang 的内存管理用的是原生 TCMalloc 模式，那么线程与内存的关系将如图所示。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651132869540-a130e8b3-1f7d-45ec-8413-52bba81426a0.png" alt="img"></p><span id="more"></span><p>ThreadCache 作为线程独立的第一交互内存，访问无需加锁，CentralCache 则作为 ThreadCache 临时补充缓存，访问需要加锁。</p><p>TCMalloc 讲对象分为三类：小对象[0,256KB] 中对象(256KB,1MB] 大对象(1MB,+∞)</p><p>ThreadCache 和 CentralCache 可以解决小对象内存块的申请,对于中/大对象的申请，TCMalloc 有一个全局共享内存堆 PageHeap。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133032054-ea888b96-0fb0-46ea-ac26-4c38abc2b66f.png" alt="image.png"></p><p>PageHeap 也是一次系统调用从虚拟内存中申请的，PageHeap 很明显是全局的，所以访问一定是要加锁。其作用是当 CentralCache 没有足够内存时会从 PageHeap 取，当 CentralCache 内存过多或者充足，则将低命中内存块退还 PageHeap。如果 Thread 需要大对象申请超过的 Cache 容纳的内存块单元大小，也会直接从 PageHeap 获取。</p><h4 id="TCMalloc-模型相关基础结构"><a href="#TCMalloc-模型相关基础结构" class="headerlink" title="TCMalloc 模型相关基础结构"></a>TCMalloc 模型相关基础结构</h4><h5 id="Page"><a href="#Page" class="headerlink" title="Page"></a>Page</h5><p>TCMalloc 将虚拟内存空间划分为多份同等大小的 Page，每个 Page 默认是 8KB。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133095495-53138cb4-89b8-4833-ac41-7957a1c19354.png" alt="image.png"></p><h5 id="Span"><a href="#Span" class="headerlink" title="Span"></a>Span</h5><p>多个连续的 Page 称之为是一个 Span。TCMalloc 是以 Span 为单位向操作系统申请内存的。Span 集合是以双向链表的形式构建。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133255704-7c07cb59-d879-468f-a925-d3494454cb7d.png" alt="image.png"></p><h5 id="Size-Class"><a href="#Size-Class" class="headerlink" title="Size Class"></a>Size Class</h5><p>在 256KB 以内的小对象，TCMalloc 会将这些小对象集合划分成多个内存刻度，同属于一个刻度类别下的内存集合称之为属于一个 Size Class</p><p>每个 Size Class 都对应一个大小比如 8KB、16KB、32KB 等。在申请小对象内存的时候，TCMalloc 会根据使用方申请的空间大小就近向上取最接近的一个 Size Class 的 Span 内存块返回给使用方。**(图中的 Size Class 单位有误)**</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133299709-8c33bad3-a31f-4844-b07c-ad54a0dc64d4.png" alt="image.png"></p><h4 id="ThreadCache"><a href="#ThreadCache" class="headerlink" title="ThreadCache"></a>ThreadCache</h4><p>在 TCMalloc 中每个线程都会有一份单独的缓存，就是 ThreadCache。ThreadCache 中对于每个 Size Class 都会有一个对应的 FreeList，FreeList 表示当前缓存中还有多少个空闲的内存可用。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133403346-3d07b578-45df-41b1-880e-d1a591d106ff.png" alt="image.png"></p><p>使用方对于从 TCMalloc 申请的小对象，会直接从 TreadCache 获取，实则是从 FreeList 中返回一个空闲的对象，如果对应的 Size Class 刻度下已经没有空闲的 Span 可以被获取了，则 ThreadCache 会从 CentralCache 中获取。当使用方使用完内存之后，归还也是直接归还给当前的 ThreadCache 中对应刻度下的的 FreeList 中。</p><p>不同 Thread 之间的 ThreadCache 是以双向链表的结构进行关联，是为了方便 TCMalloc 统计和管理。</p><h4 id="CentralCache"><a href="#CentralCache" class="headerlink" title="CentralCache"></a>CentralCache</h4><p>CentralCache 是各个线程共用的，所以与 CentralCache 获取内存交互是需要加锁的。CentralCache 缓存的 Size Class 和 ThreadCache 的一样，这些缓存都被放在 CentralFreeList 中，当 ThreadCache 中的某个 Size Class 刻度下的缓存小对象不够用，就会向 CentralCache 对应的 Size Class 刻度的 CentralFreeList 获取，同样的如果 ThreadCache 有多余的缓存对象也会退还给响应的 CentralFreeList。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133530033-bd9265dc-fd49-4a77-a845-f175ab317ea9.png" alt="image.png"></p><h4 id="PageHeap"><a href="#PageHeap" class="headerlink" title="PageHeap"></a>PageHeap</h4><p>PageHead 与 CentralCache 不同的是 CentralCache 是与 ThreadCache 布局一模一样的缓存，主要是起到针对 ThreadCache 的一层二级缓存作用，且只支持小对象内存分配。而 PageHeap 则是针对 CentralCache 的三级缓存。弥补对于中对象内存和大对象内存的分配，PageHeap 也是直接和操作系统虚拟内存衔接的一层缓存，当 ThreadCache、CentralCache、PageHeap 都找不到合适的 Span，PageHeap 则会调用操作系统内存申请系统调用函数来从虚拟内存的堆区中取出内存填充到 PageHeap 当中。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133596465-fd16a3cc-256a-464c-a066-b896043a9f63.png" alt="image.png"></p><p>PageHeap 内部的 Span 管理，采用两种不同的方式，对于 128 个 Page 以内的 Span 申请，每个 Page 刻度都会用一个链表形式的缓存来存储。对于 128 个 Page 以上内存申请，PageHeap 是以有序集合来存放。</p><h4 id="TCMalloc-的小对象分配"><a href="#TCMalloc-的小对象分配" class="headerlink" title="TCMalloc 的小对象分配"></a>TCMalloc 的小对象分配</h4><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133672724-0ac13b26-1623-444a-8c81-0b2120b2e2fa.png" alt="image.png"></p><h4 id="TCMalloc-的中对象分配"><a href="#TCMalloc-的中对象分配" class="headerlink" title="TCMalloc 的中对象分配"></a>TCMalloc 的中对象分配</h4><p>中对象为大于 256KB 且小于等于 1MB 的内存。对于中对象申请分配的流程 TCMalloc 与处理小对象分配有一定的区别。对于中对象分配，Thread 不再按照小对象的流程路径向 ThreadCache 获取，而是直接从 PageHeap 获取，具体的流程如图所示。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133803693-486e9b4a-ffb1-4932-a989-1df013b601c1.png" alt="image.png"></p><p>PageHeap 将 128 个 Page 以内大小的 Span 定义为小 Span，将 128 个 Page 以上大小的 Span 定义为大 Span。由于一个 Page 为 8KB，那么 128 个 Page 即为 1MB，所以对于中对象的申请，PageHeap 均是按照小 Span 的申请流程。</p><h4 id="TCMalloc-的大对象分配"><a href="#TCMalloc-的大对象分配" class="headerlink" title="TCMalloc 的大对象分配"></a>TCMalloc 的大对象分配</h4><p>对于超过 128 个 Page（即 1MB）的内存分配则为大对象分配流程。大对象分配与中对象分配情况类似，Thread 绕过 ThreadCache 和 CentralCache，直接向 PageHeap 获取。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651133987470-28f3feb2-8a9e-45be-a41b-596b1bd54e8d.png" alt="image.png"></p><h3 id="Golang-堆内存管理"><a href="#Golang-堆内存管理" class="headerlink" title="Golang 堆内存管理"></a>Golang 堆内存管理</h3><p>Golang 内存管理模型的逻辑层次全景图，如图所示。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651134285363-999c7495-7834-4785-a6ea-c44b4615ff19.png" alt="image.png"></p><h4 id="Golang-内存管理单元相关概念"><a href="#Golang-内存管理单元相关概念" class="headerlink" title="Golang 内存管理单元相关概念"></a>Golang 内存管理单元相关概念</h4><h5 id="Page-1"><a href="#Page-1" class="headerlink" title="Page"></a>Page</h5><p>与 TCMalloc 的 Page 一致。</p><h5 id="mSpan"><a href="#mSpan" class="headerlink" title="mSpan"></a>mSpan</h5><p>与 TCMalloc 中的 Span 一致。</p><h5 id="Size-Class-1"><a href="#Size-Class-1" class="headerlink" title="Size Class"></a>Size Class</h5><p>Golang 内存管理针对 Size Class 对衡量内存的概念更加详细，这里面介绍一些基础的有关内存大小的名词及算法。</p><ul><li>Object Size: 应用一次向 Golang 内存申请的对象 Object 大小。 Page 是 Golang 内存管理与操作系统交互衡量内存容量的基本单元，Golang 内存管理内部本身用来给对象存储内存的基本单元是 Object。</li></ul><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651134337384-3b5b18a9-63a2-41eb-89fb-d7a030b1e569-20220904121240445.png" alt="image.png"></p><ul><li>Size Class</li></ul><p>Golang 内存管理中的 Size Class 与 TCMalloc 所表示的设计含义是一致的，都表示一块内存的所属规格或者刻度。Golang 内存管理中的 Size Class 是针对 Object Size 来划分内存的。</p><ul><li>Span Class</li></ul><p>这个是 Golang 内存管理额外定义的规格属性，是针对 Span 来进行划分的，是 Span 大小的级别。<strong>一个 Size Class 会对应两个 Span Class</strong>，其中一个 Span 为存放需要 GC 扫描的对象（包含指针的对象），另一个 Span 为存放不需要 GC 扫描的对象（不包含指针的对象）</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651134377320-3f71752d-65fa-4081-a255-09c387f23a65.png" alt="image.png"></p><h4 id="MCache"><a href="#MCache" class="headerlink" title="MCache"></a>MCache</h4><p>从概念来讲 MCache 与 TCMalloc 的 ThreadCache 十分相似，访问 mcache 依然不需要加锁而是直接访问，且 MCache 中依然保存各种大小的 Span。虽然 MCache 与 ThreadCache 概念相似，二者还是存在一定的区别的，<strong>MCache 是与 Golang 协程调度模型 GPM 中的 P 所绑定，而不是和线程绑定</strong>。MCache 与 P 进行绑定更能节省内存空间使用，可以保证每个 G 使用 MCache 时不需要加锁就可以获取到内存。而 TCMalloc 中的 ThreadCache 随着 Thread 的增多。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651134640677-2a153c96-b7e8-46bc-86f3-dfaf50087329.png" alt="image.png"></p><p>对于 Span Class 为 0 和 1 的，也就是对应 Size Class 为 0 的规格刻度内存，mcache 实际上是没有分配任何内存的。因为 Golang 内存管理对内存为 0 的数据申请做了特殊处理，<strong>如果申请的数据大小为 0 将直接返回一个固定内存地址</strong>，不会走 Golang 内存管理的正常逻辑</p><h4 id="MCentral"><a href="#MCentral" class="headerlink" title="MCentral"></a>MCentral</h4><p>当 MCache 中出现 Size Class 的 Span 空缺情况，MCache 则会向 MCentral 申请对应的 Span。Goroutine、MCache、MCentral、MHeap 互相交换的内存单位是不同，具体如图所示。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651134740690-c1fc15a5-af2a-474c-adaa-ddc4bb05a5e3.png" alt="image.png"></p><p>MCentral 与 TCMalloc 中的 Central 不同的是 MCentral 针对每个 Span Class 级别有两个 Span 链表，而 TCMalloc 中的 Central 只有一个。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651134816735-43c615ce-3c3c-485a-9ae2-e36dca963f95.png" alt="image.png"></p><p>MCentral 与 MCache 不同的是，每个级别保存的不是<strong>一个 Span</strong>，而是<strong>一个 Span List 链表</strong>。与 TCMalloc 中的 Central 不同的是，MCentral 每个级别都保存了<strong>两个 Span List</strong>。</p><h4 id="MHeap"><a href="#MHeap" class="headerlink" title="MHeap"></a>MHeap</h4><p>MHeap 是对内存块的管理对象，是通过 Page 为内存单元进行管理。那么用来详细管理每一系列 Page 的结构称之为一个 HeapArena，它们的逻辑层级关系如图所示。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651135097610-8b18c759-6207-435d-80d4-616ce62de8d3.png" alt="image.png"></p><p>一个 HeapArena 占用内存 64MB，其中里面的内存的是一个一个的 mspan，当然最小单元依然是 Page，图中没有表示出 mspan，因为多个连续的 page 就是一个 mspan。所有的 HeapArena 组成的集合是一个 Arenas，也就是 MHeap 针对堆内存的管理。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651135153113-4db62f09-063b-4470-9fa9-1229150f703c.png" alt="image.png"></p><h4 id="Tiny-对象分配流程"><a href="#Tiny-对象分配流程" class="headerlink" title="Tiny 对象分配流程"></a>Tiny 对象分配流程</h4><p>Golang 内存管理将对象大小进行了分类：Tiny 对象/小对象(16B 至 32KB)/大对象</p><p>针对 Tiny 微小对象的分配，实际上 Golang 做了比较特殊的处理，之前在介绍 MCache 的时候并没有提及有关 Tiny 的存储和分配问题，MCache 中不仅保存着各个 Span Class 级别的内存块空间，还有一个比较特殊的 Tiny 存储空间。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651135206587-7442cb63-77a7-4b87-8822-db91367db164.png" alt="image.png"></p><p>Tiny 空间是从 Size Class = 2（对应 Span Class = 4 或 5）中获取一个 16B 的 Object，作为 Tiny 对象的分配空间。对于 Golang 内存管理为什么需要一个 Tiny 这样的 16B 空间，原因是因为如果协程逻辑层申请的内存空间小于等于 8B，那么根据正常的 Size Class 匹配会匹配到 Size Class = 1（对应 Span Class = 2 或 3），所以像 int32、 byte、 bool 以及小字符串等经常使用的 Tiny 微小对象，也都会使用从 Size Class = 1 申请的这 8B 的空间。但是类似 bool 或者 1 个字节的 byte，也都会各自独享这 8B 的空间，进而导致有一定的内存空间浪费。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651135299397-38b04b81-3179-44e4-81ca-bb476b69f22f.png" alt="image.png"></p><h4 id="小对象分配流程"><a href="#小对象分配流程" class="headerlink" title="小对象分配流程"></a>小对象分配流程</h4><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651135402859-c6404c6a-f0fd-4bb9-bbef-0c5286b0b2a4.png" alt="image.png"></p><h4 id="大对象分配流程"><a href="#大对象分配流程" class="headerlink" title="大对象分配流程"></a>大对象分配流程</h4><p>小对象是在 MCache 中分配的，而大对象是直接从 MHeap 中分配。对于不满足 MCache 分配范围的对象，均是按照大对象分配流程处理。</p><p>大对象分配流程是协程逻辑层直接向 MHeap 申请对象所需要的适当 Pages，从而绕过从 MCaceh 到 MCentral 的繁琐申请内存流程，大对象的内存分配流程相对比较简单，具体的流程如图所示。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651135463380-9ee93382-7deb-48c0-ab38-519d679101e4.png" alt="image.png"></p>]]></content>
<summary type="html"><h2 id="内存管理"><a href="#内存管理" class="headerlink" title="内存管理"></a>内存管理</h2><h3 id="TCMalloc"><a href="#TCMalloc" class="headerlink" title="TCMalloc"></a>TCMalloc</h3><p>TCMalloc 为每个 Thread 预分配一块缓存，每个 Thread 在申请内存时首先会先从这个缓存区 ThreadCache 申请，且所有 ThreadCache 缓存区还共享一个叫 CentralCache 的中心缓存。这里假设目前 Golang 的内存管理用的是原生 TCMalloc 模式，那么线程与内存的关系将如图所示。</p>
<p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/1651132869540-a130e8b3-1f7d-45ec-8413-52bba81426a0.png" alt="img"></p></summary>
<category term="go" scheme="https://cwww3.github.io/tags/go/"/>
<category term="memory" scheme="https://cwww3.github.io/tags/memory/"/>
</entry>
<entry>
<title>Informer机制</title>
<link href="https://cwww3.github.io/2022/09/04/informer/"/>
<id>https://cwww3.github.io/2022/09/04/informer/</id>
<published>2022-09-03T18:15:50.000Z</published>
<updated>2022-09-03T18:17:14.531Z</updated>
<content type="html"><![CDATA[<p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/watermark%2Ctype_ZmFuZ3poZW5naGVpdGk%2Cshadow_10%2Ctext_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTMyNzYyNzc%3D%2Csize_16%2Ccolor_FFFFFF%2Ct_70.png" alt="在这里插入图片描述"></p><span id="more"></span><p>在 Informer 架构中，有多个核心组件</p><h3 id="Reflector"><a href="#Reflector" class="headerlink" title="Reflector"></a>Reflector</h3><p>Reflector 是用于监控指定资源的(Watch)，一旦资源发生改变，将资源存入 DeltaFIFO 中。</p><p>在第一次运行时，会先获取一份完成的资源列表(List)存入 DeltaFIFO 中。</p><h3 id="DeltaFIFO"><a href="#DeltaFIFO" class="headerlink" title="DeltaFIFO"></a>DeltaFIFO</h3><p>DeltaFIFO 可以分开理解，首先 FIFO 是一个先进先出的队列，拥有操作队列的基本方法，Delta 则是存储资源对象以及对象的操作类型(Add/Delete/Update/Sync)，DeltaFIFO 与其他队列最大的不同则是它会保留所有资源对象的操作类型，即队列中会存在拥有不同操作类型的同一个资源对象。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> DeltaFIFO <span class="keyword">struct</span> {</span><br><span class="line"> ...</span><br><span class="line"> queue []<span class="keyword">string</span> <span class="comment">// 存储的是key,资源对象通过KeyOf生成key，key与资源对象对应</span></span><br><span class="line"> items <span class="keyword">map</span>[<span class="keyword">string</span>]Deltas <span class="comment">// map的key就是queue中的元素</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">type</span> Deltas []Delta <span class="comment">// Delta 保存的是资源对象以及操作类型 Deltas中每个资源的操作类型都不同(去重)</span></span><br></pre></td></tr></table></figure><p>资源从队列中被 Pop 出之后，会执行 informer 会调用 HandleDeltas 方法，将它加入到 indexder 中，此外，还会将它分发给用户通过 Informer 的 AddEventHandler 方法注册的回调函数中。</p><h4 id="Resync"><a href="#Resync" class="headerlink" title="Resync"></a>Resync</h4><p>Resync 机制会将 Indexer 中存储的资源对象同步到 DeltaFIFO 中，并将这些资源对象设置为 Sync 的操作类型。</p><p>Resync 函数在 Reflector 中定时执行，执行周期由创建 informer 时设定。</p><h3 id="Indexer"><a href="#Indexer" class="headerlink" title="Indexer"></a>Indexer</h3><p>indexer 是 client-go 用来存储资源对象并自带索引功能的本地存储。Reflector 从 DeltaFIFO 中取出对象存储至 Indexer。Indexer 与 Etcd 中所保存的数据完全一致，client-go 可以很方便的从本地获取数据，而不需要每次请求 API Server 从 Etcd 中获取数据,从而减轻了 API Server 和 Etcd 的压力。</p><p>Indexer 是在 ThreadSafeMap 的基础上进行封装的。ThreadSafeMap 是一个并发读写安全的 map，Indexer 在此基础上实现了索引功能。</p><p>Index 有四个非常重要的数据结构，分别是 Indices(index 的复数形式),index,Indexers 和 IndexerFunc。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> Indexers <span class="keyword">map</span>[<span class="keyword">string</span>]IndexFunc</span><br><span class="line"><span class="keyword">type</span> IndexFunc <span class="function"><span class="keyword">func</span><span class="params">(obj <span class="keyword">interface</span>{})</span> <span class="params">([]<span class="keyword">string</span>, error)</span></span></span><br><span class="line"><span class="keyword">type</span> Indices <span class="keyword">map</span>[<span class="keyword">string</span>]Index</span><br><span class="line"><span class="keyword">type</span> Index <span class="keyword">map</span>[<span class="keyword">string</span>]sets.String</span><br></pre></td></tr></table></figure><p><strong>Indexers</strong>:用于存储索引器，key 为索引器的名称，value 位索引器的实现函数，它定义了获取资源对象的什么信息,以及如何获取,并组装成字符串数组返回。</p><p><strong>IndexFunc</strong>: 索引器函数，定义为接收一个资源对象，返回检索结果列表</p><p><strong>Indeces</strong>: 用于存储 Index，key 为索引器的名字，value 为 Index</p><p><strong>index</strong>: 用于存储缓存的数据</p><p>index.ByIndex 函数通过执行索引器函数得到索引结果。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(c *threadSafeMap)</span> <span class="title">ByIndex</span><span class="params">(indexName, indexKey <span class="keyword">string</span>)</span> <span class="params">([]<span class="keyword">interface</span>()</span>,<span class="title">error</span>)</span></span><br><span class="line">{</span><br><span class="line"> ...</span><br><span class="line"> indexFunc := c.indexers[indexName] <span class="comment">// 根据索引器名称得到索引器函数</span></span><br><span class="line"> ... <span class="comment">// 执行该函数</span></span><br><span class="line"></span><br><span class="line"> index := c.indices[indexName] <span class="comment">// 根据索引器名称从indices中获取index</span></span><br><span class="line"> set := index[indexKey] <span class="comment">// 根据查询的key得到查询的结果</span></span><br><span class="line"> ...</span><br><span class="line">}</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/watermark%2Ctype_ZmFuZ3poZW5naGVpdGk%2Cshadow_10%2Ctext_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTMyNzYyNzc%3D%2Csize_16%2Ccolor_FFFFFF%2Ct_70.png" alt="在这里插入图片描述"></p></summary>
<category term="k8s" scheme="https://cwww3.github.io/tags/k8s/"/>
</entry>
<entry>
<title>processAndthread</title>
<link href="https://cwww3.github.io/2022/07/23/processAndthread/"/>
<id>https://cwww3.github.io/2022/07/23/processAndthread/</id>
<published>2022-07-23T03:55:03.000Z</published>
<updated>2022-07-23T03:55:19.150Z</updated>
<content type="html"><![CDATA[<h2 id="进程"><a href="#进程" class="headerlink" title="进程"></a>进程</h2><h3 id="状态"><a href="#状态" class="headerlink" title="状态"></a>状态</h3><p>创建 就绪 运行 阻塞 阻塞挂起 就绪挂起 结束 </p><p>程序阻塞时，会将程序所占用的物理内存,swap到硬盘上，此时进程进入了挂起状态</p><p>程序sleep也会导致挂起</p><span id="more"></span><h3 id="进程控制块（PCB）"><a href="#进程控制块（PCB）" class="headerlink" title="进程控制块（PCB）"></a>进程控制块（PCB）</h3><h4 id="进程描述信息"><a href="#进程描述信息" class="headerlink" title="进程描述信息"></a>进程描述信息</h4><p>进程标识符 用户标识符</p><h4 id="进程控制和管理信息"><a href="#进程控制和管理信息" class="headerlink" title="进程控制和管理信息"></a>进程控制和管理信息</h4><p>进程当前状态 进程的优先级</p><h4 id="资源分配清单"><a href="#资源分配清单" class="headerlink" title="资源分配清单"></a>资源分配清单</h4><p>有关内存地址空间或虚拟地址空间的信息</p><p>所打开的文件的列表，所使用的的I/O设备信息</p><h4 id="CPU相关信息"><a href="#CPU相关信息" class="headerlink" title="CPU相关信息"></a>CPU相关信息</h4><p>CPU中各个寄存器的值，当进程切换时，CPU的状态信息都会被保存在相应地PCB中，以便程序能从断点出继续执行</p><h2 id="线程"><a href="#线程" class="headerlink" title="线程"></a>线程</h2><h3 id="线程控制块（TCP）"><a href="#线程控制块（TCP）" class="headerlink" title="线程控制块（TCP）"></a>线程控制块（TCP）</h3><p>一个进程可以存在多个线程，线程之间共享进程的资源，除了维护自己所必需的栈以及寄存器。</p><h3 id="协程"><a href="#协程" class="headerlink" title="协程"></a>协程</h3><p>用户态线程，在用户态空间实现的线程，不是由内核管理的线程，是由用户态的线程库来完成线程的管理</p><h2 id="CPU上下文切换"><a href="#CPU上下文切换" class="headerlink" title="CPU上下文切换"></a>CPU上下文切换</h2><p>CPU的上下文即程序计数器以及寄存器，程序计数器用来存储CPU正在执行的指令位置</p><p>CPU在执行任务前，需要设置好寄存器以及程序计数器</p><p>CPU上下文切换就是把前一个任务的CPU上下文保存起来，然后加载新任务，更新上下文信息。</p><p>任务即进程、线程和中断。</p><h3 id="进程上下文切换"><a href="#进程上下文切换" class="headerlink" title="进程上下文切换"></a>进程上下文切换</h3><p>进程的上下文切换不仅包含了虚拟内存、栈、全局变量等用户空间的资源，也包含了内核堆栈、寄存器等内核空间的资源</p><h3 id="线程上下文切换"><a href="#线程上下文切换" class="headerlink" title="线程上下文切换"></a>线程上下文切换</h3><p>线程是进程当中的一条执行流程。</p><p>同一个进程内的多个线程可以共享代码段、数据段、打开的文件资源等，但每个线程各自都有一套独立的寄存器和栈，这样可以确保线程的控制流是相对独立的。</p><p>只需要切换线程的私有数据、寄存器等不共享的数据。</p><h4 id="比较"><a href="#比较" class="headerlink" title="比较"></a>比较</h4><p>进程是资源分配的单位，线程是CPU调度的单位</p><h3 id="调度算法"><a href="#调度算法" class="headerlink" title="调度算法"></a>调度算法</h3><h4 id="先来先服务"><a href="#先来先服务" class="headerlink" title="先来先服务"></a>先来先服务</h4><h4 id="最短作业有限"><a href="#最短作业有限" class="headerlink" title="最短作业有限"></a>最短作业有限</h4><h4 id="高响应比优先"><a href="#高响应比优先" class="headerlink" title="高响应比优先"></a>高响应比优先</h4><h4 id="时间片轮转"><a href="#时间片轮转" class="headerlink" title="时间片轮转"></a>时间片轮转</h4><h4 id="最高优先级"><a href="#最高优先级" class="headerlink" title="最高优先级"></a>最高优先级</h4><h4 id="多级反馈队列"><a href="#多级反馈队列" class="headerlink" title="多级反馈队列"></a>多级反馈队列</h4><p>抢占式/非抢占式</p><h2 id="进程间通信"><a href="#进程间通信" class="headerlink" title="进程间通信"></a>进程间通信</h2><ol><li>管道</li><li>消息队列</li><li>共享内存(虚拟内存映射相同的物理地址)</li><li>信号量(P/V)</li><li>信号(kill)</li><li>socket</li></ol><h2 id="同步与互斥"><a href="#同步与互斥" class="headerlink" title="同步与互斥"></a>同步与互斥</h2><h4 id="同步"><a href="#同步" class="headerlink" title="同步"></a>同步</h4><p>并发进程/线程在一些关键点上可能需要互相等待与互通消息，这种制约的等待与互通信息称为进程/线程同步。如操作A应在操作B之前执行…</p><h4 id="互斥"><a href="#互斥" class="headerlink" title="互斥"></a>互斥</h4><p>如操作A与操作B不能在同一时刻执行…</p><h4 id="实现"><a href="#实现" class="headerlink" title="实现"></a>实现</h4><p>锁：加锁/解锁</p><p>信号量：P/V操作 </p><p>都可以实现互斥，信号量还可以实现同步</p><blockquote><p>哲学家就餐问题</p></blockquote><h4 id="死锁"><a href="#死锁" class="headerlink" title="死锁"></a>死锁</h4><p>死锁的四个必要条件：</p><ul><li>互斥</li><li>持有并等待</li><li>不可剥夺</li><li>环路</li></ul><p>解决死锁就是破坏这一个或多个条件。</p><p>最基本的锁有两种：</p><ol><li>互斥锁，抢锁失败后，线程释放CPU(阻塞，用户态陷入内核态)</li><li>自旋锁，抢锁失败后，线程忙等待，知道拿到锁</li></ol><p>上下文切换需要时间成本，如果锁住的代码执行时间较短，应该使用自旋锁。</p><h5 id="读写锁"><a href="#读写锁" class="headerlink" title="读写锁"></a>读写锁</h5><h5 id="乐观锁-悲观锁"><a href="#乐观锁-悲观锁" class="headerlink" title="乐观锁 悲观锁"></a>乐观锁 悲观锁</h5><p>上面所提到的锁都是悲观锁</p><p>乐观锁全称并没有加锁，也叫无锁编程(在线文档)，虽然祛除了加锁解锁的操作，但是一旦发生冲突，重试的成本非常高，适合在冲突概率低，且加锁成本非常高的场景。</p>]]></content>
<summary type="html"><h2 id="进程"><a href="#进程" class="headerlink" title="进程"></a>进程</h2><h3 id="状态"><a href="#状态" class="headerlink" title="状态"></a>状态</h3><p>创建 就绪 运行 阻塞 阻塞挂起 就绪挂起 结束 </p>
<p>程序阻塞时，会将程序所占用的物理内存,swap到硬盘上，此时进程进入了挂起状态</p>
<p>程序sleep也会导致挂起</p></summary>
<category term="os" scheme="https://cwww3.github.io/tags/os/"/>
</entry>
<entry>
<title>memory</title>
<link href="https://cwww3.github.io/2022/07/23/memory/"/>
<id>https://cwww3.github.io/2022/07/23/memory/</id>
<published>2022-07-23T03:54:08.000Z</published>
<updated>2022-09-03T18:24:40.803Z</updated>
<content type="html"><![CDATA[<p>物理内存，大小有限，多个程序使用，不易分配。</p><p>引入虚拟地址，程序的视角认为整个内存只有它在使用。程序间相互隔离，互不影响。</p><p>CPU 中的内存管理单元(MMU)，负责将虚拟地址转化为实际的物理地址。</p><span id="more"></span><h3 id="管理虚拟地址和物理地址"><a href="#管理虚拟地址和物理地址" class="headerlink" title="管理虚拟地址和物理地址"></a>管理虚拟地址和物理地址</h3><h4 id="分段"><a href="#分段" class="headerlink" title="分段"></a>分段</h4><p>程序由若干个逻辑分段组成，代码分段、数据分段、栈段、堆段组成。虚拟地址由两部分组成，<strong>段选择因子</strong>和<strong>段内偏移量</strong>。根据段选择因子可以从存储在内存中的段表中找到<strong>段基地址</strong>，再根据段内偏移量，得到真正的物理地址。段选择因子还包含了其他信息,段界限(限制段的大小),特权级 DPL(读/写控制)</p><h5 id="问题："><a href="#问题：" class="headerlink" title="问题："></a>问题：</h5><p>内存碎片,会产生多个不连续的物理内存，导致新的程序无法被装载。</p><p>内存浪费，程序的所有内存都被装载到物理内存，但是有部分内存并不常用。</p><h5 id="解决："><a href="#解决：" class="headerlink" title="解决："></a>解决：</h5><p>内存交换，对于不连续的内存，可以读入到磁盘，再从磁盘中读出，连续地存入内存(Swap)</p><h5 id="问题：-1"><a href="#问题：-1" class="headerlink" title="问题："></a>问题：</h5><p>如果频繁的进行内存交换，且交换的是内存占用很大的程序，那么效率肯定会很低。引入分页解决</p><h4 id="分页"><a href="#分页" class="headerlink" title="分页"></a>分页</h4><p>优化交换的内存大小，让需要交换写入或者从磁盘装载的数据更少一点。</p><p>分页是把<strong>整个虚拟和物理内存空间</strong>切成一段段固定尺寸的大小——页(Page).虚拟地址和物理地址的映射通过存储在内存中的页表来映射。<strong>页表保存所有虚拟地址和物理地址的映射</strong></p><p>如果内存空间不够，操作系统会把其他正在运行的进程中的最近没被使用的内存页面给释放掉，暂时暂时卸载硬盘上(Swap Out)，一旦需要，再加载进来(Swap In)。每次交换的大小就是一个页或者几个页，提升了交换的效率。</p><p>分页使得我们在加载程序的时候，不再需要一次性把程序加载到物理内存中，而只有在程序运行中，需要用到对应虚拟内存页里面的指令和数据时，再加载到物理内存里。</p><p>在分页机制下，虚拟地址分为<strong>页号</strong>和<strong>业内偏移</strong>，根据页号，能从页表中获取<strong>物理内存的基地址</strong>，再根据业内偏移得到物理内存地址。</p><h5 id="问题：-2"><a href="#问题：-2" class="headerlink" title="问题："></a>问题：</h5><p>因为页表存储的是整个虚拟地址和物理地址的映射，那么存储一个页表所需的内存会比较多，且每一个程序都会拥有一个自己的页表，那会导致很大一部分内存用于存储页表信息。</p><h5 id="解决：-1"><a href="#解决：-1" class="headerlink" title="解决："></a>解决：</h5><p>多级页表(Multi-Level Page Table)</p><p>一级页表中存储 1024 个页表项(覆盖整个虚拟空间的大小，相比原来一级分页中的页表空间占用小了很多)，根据页表项中的信息，可以得到二级页表中的页表项，再根据二级页表中的页表项得到物理地址。</p><p>根据<strong>局部性原理</strong>，每个进程都有 4GB 的虚拟空间，显然，其使用的空间远未达到 4GB，因为存在部分的页表项是空的，根本没有分配，对于已分配的页表项，如果一定时间未访问，在物理内存紧张的情况下，也会被换出到因硬盘。</p><p>所以如果使用了二级分页，<strong>如果一级页表的页表项没有被分配，就不需要创建二级页表了</strong>，这样就能减少页表对内存的占用。</p><h5 id="问题：-3"><a href="#问题：-3" class="headerlink" title="问题："></a>问题：</h5><p>虽然在空间上减少了占用，但是由于采用了多级映射的原因，导致地址转换的速度变慢了。</p><h5 id="解决：-2"><a href="#解决：-2" class="headerlink" title="解决："></a>解决：</h5><p>还是程序局部性原理，在一段时间内，整个程序的执行仅限于程序中的某一部分。相应地，执行所访问的存储空间也局限于某个内存区域。利用这一特性，科学家在 CPU 芯片中，加入了一个专门存放程序最常访问的页表项的 Cache——TLB.</p><h4 id="段页式"><a href="#段页式" class="headerlink" title="段页式"></a>段页式</h4><p>分段和分页并不是对立的，可以组合起来使用。</p><ol><li>现将程序划分为多个有逻辑意义的段</li><li>再将每个段分为多个页</li></ol><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/2022-09-02-20-03-49-image.png"></p>]]></content>
<summary type="html"><p>物理内存，大小有限，多个程序使用，不易分配。</p>
<p>引入虚拟地址，程序的视角认为整个内存只有它在使用。程序间相互隔离，互不影响。</p>
<p>CPU 中的内存管理单元(MMU)，负责将虚拟地址转化为实际的物理地址。</p></summary>
<category term="os" scheme="https://cwww3.github.io/tags/os/"/>
</entry>
<entry>
<title>go-sql</title>
<link href="https://cwww3.github.io/2022/07/05/go-sql/"/>
<id>https://cwww3.github.io/2022/07/05/go-sql/</id>
<published>2022-07-05T15:58:13.000Z</published>
<updated>2022-07-05T16:01:28.213Z</updated>
<content type="html"><![CDATA[<h2 id="Sql源码浅析"><a href="#Sql源码浅析" class="headerlink" title="Sql源码浅析"></a>Sql源码浅析</h2><h3 id="Connect"><a href="#Connect" class="headerlink" title="Connect"></a>Connect</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">Open</span><span class="params">(driverName, dataSourceName <span class="keyword">string</span>)</span> <span class="params">(*DB, error)</span></span> {</span><br><span class="line"><span class="comment">// 查找已注册的驱动 由_ "github.com/go-sql-driver/mysql"提供</span></span><br><span class="line"> <span class="comment">// 驱动是真正和数据库打交道的</span></span><br><span class="line"> driversMu.RLock()</span><br><span class="line">driveri, ok := drivers[driverName]</span><br><span class="line">driversMu.RUnlock()</span><br><span class="line"><span class="keyword">if</span> !ok {</span><br><span class="line"><span class="keyword">return</span> <span class="literal">nil</span>, fmt.Errorf(<span class="string">"sql: unknown driver %q (forgotten import?)"</span>, driverName)</span><br><span class="line">}</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> driverCtx, ok := driveri.(driver.DriverContext); ok {</span><br><span class="line"> <span class="comment">// 获取连接器</span></span><br><span class="line">connector, err := driverCtx.OpenConnector(dataSourceName)</span><br><span class="line"><span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span> <span class="literal">nil</span>, err</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> OpenDB(connector), <span class="literal">nil</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span> OpenDB(dsnConnector{dsn: dataSourceName, driver: driveri}), <span class="literal">nil</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// OpenDB may just validate its arguments without creating a connection</span></span><br><span class="line"><span class="comment">// to the database. To verify that the data source name is valid, call</span></span><br><span class="line"><span class="comment">// Ping.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">OpenDB</span><span class="params">(c driver.Connector)</span> *<span class="title">DB</span></span> {</span><br><span class="line"> <span class="comment">// 初始化数据，为后续维护连接池做准备</span></span><br><span class="line">ctx, cancel := context.WithCancel(context.Background())</span><br><span class="line">db := &DB{</span><br><span class="line">connector: c,</span><br><span class="line">openerCh: <span class="built_in">make</span>(<span class="keyword">chan</span> <span class="keyword">struct</span>{}, connectionRequestQueueSize),</span><br><span class="line">lastPut: <span class="built_in">make</span>(<span class="keyword">map</span>[*driverConn]<span class="keyword">string</span>),</span><br><span class="line">connRequests: <span class="built_in">make</span>(<span class="keyword">map</span>[<span class="keyword">uint64</span>]<span class="keyword">chan</span> connRequest),</span><br><span class="line">stop: cancel,</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">go</span> db.connectionOpener(ctx)</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span> db</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// Runs in a separate goroutine, opens new connections when requested.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">connectionOpener</span><span class="params">(ctx context.Context)</span></span> {</span><br><span class="line"><span class="keyword">for</span> {</span><br><span class="line"><span class="keyword">select</span> {</span><br><span class="line"><span class="keyword">case</span> <-ctx.Done():</span><br><span class="line"><span class="keyword">return</span></span><br><span class="line"><span class="keyword">case</span> <-db.openerCh:</span><br><span class="line"> <span class="comment">// 收到创建新的连接的请求</span></span><br><span class="line">db.openNewConnection(ctx)</span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line">}</span><br></pre></td></tr></table></figure><span id="more"></span><h3 id="Tx"><a href="#Tx" class="headerlink" title="Tx"></a>Tx</h3><h4 id="Begin"><a href="#Begin" class="headerlink" title="Begin"></a>Begin</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">Begin</span><span class="params">()</span> <span class="params">(*Tx, error)</span></span> {</span><br><span class="line"><span class="keyword">return</span> db.BeginTx(context.Background(), <span class="literal">nil</span>)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">BeginTx</span><span class="params">(ctx context.Context, opts *TxOptions)</span> <span class="params">(*Tx, error)</span></span> {</span><br><span class="line"><span class="keyword">var</span> tx *Tx</span><br><span class="line"><span class="keyword">var</span> err error</span><br><span class="line"><span class="keyword">var</span> isBadConn <span class="keyword">bool</span></span><br><span class="line"> <span class="comment">// 重试策略</span></span><br><span class="line"><span class="keyword">for</span> i := <span class="number">0</span>; i < maxBadConnRetries; i++ {</span><br><span class="line"> <span class="comment">// cachedOrNewConn 获取连接池的连接，如果没有，</span></span><br><span class="line"> <span class="comment">// 在连接没有到达设置的最大值时，就创建一个新的连接, 否则只能等待。</span></span><br><span class="line"> <span class="comment">// inUse=true</span></span><br><span class="line">tx, err = db.begin(ctx, opts, cachedOrNewConn)</span><br><span class="line">isBadConn = errors.Is(err, driver.ErrBadConn)</span><br><span class="line"><span class="keyword">if</span> !isBadConn {</span><br><span class="line"><span class="keyword">break</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"><span class="keyword">if</span> isBadConn {</span><br><span class="line"><span class="keyword">return</span> db.begin(ctx, opts, alwaysNewConn)</span><br><span class="line">}</span><br><span class="line"><span class="keyword">return</span> tx, err</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">begin</span><span class="params">(ctx context.Context, opts *TxOptions, strategy connReuseStrategy)</span> <span class="params">(tx *Tx, err error)</span></span> {</span><br><span class="line">dc, err := db.conn(ctx, strategy)</span><br><span class="line"><span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span> <span class="literal">nil</span>, err</span><br><span class="line">}</span><br><span class="line"> <span class="comment">// releaseConn释放连接时调用,放入连接池 inUse=false</span></span><br><span class="line"><span class="keyword">return</span> db.beginDC(ctx, dc, dc.releaseConn, opts)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">begin</span><span class="params">(ctx context.Context, opts *TxOptions, strategy connReuseStrategy)</span> <span class="params">(tx *Tx, err error)</span></span> {</span><br><span class="line"> <span class="comment">// 根据策略获取到了dc</span></span><br><span class="line">dc, err := db.conn(ctx, strategy)</span><br><span class="line"><span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span> <span class="literal">nil</span>, err</span><br><span class="line">}</span><br><span class="line"> <span class="comment">// 传入dc</span></span><br><span class="line"><span class="keyword">return</span> db.beginDC(ctx, dc, dc.releaseConn, opts)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// beginDC starts a transaction. The provided dc must be valid and ready to use.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">beginDC</span><span class="params">(ctx context.Context, dc *driverConn, release <span class="keyword">func</span>(error)</span>, <span class="title">opts</span> *<span class="title">TxOptions</span>) <span class="params">(tx *Tx, err error)</span></span> {</span><br><span class="line"><span class="keyword">var</span> txi driver.Tx</span><br><span class="line">keepConnOnRollback := <span class="literal">false</span></span><br><span class="line">withLock(dc, <span class="function"><span class="keyword">func</span><span class="params">()</span></span> {</span><br><span class="line">_, hasSessionResetter := dc.ci.(driver.SessionResetter)</span><br><span class="line">_, hasConnectionValidator := dc.ci.(driver.Validator)</span><br><span class="line">keepConnOnRollback = hasSessionResetter && hasConnectionValidator</span><br><span class="line"> <span class="comment">// dc.ci就是驱动与数据库建立连接的原始结构,通过它开启事务，返回事务的原始结构</span></span><br><span class="line">txi, err = ctxDriverBegin(ctx, opts, dc.ci)</span><br><span class="line">})</span><br><span class="line"><span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line">release(err)</span><br><span class="line"><span class="keyword">return</span> <span class="literal">nil</span>, err</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// Schedule the transaction to rollback when the context is canceled.</span></span><br><span class="line"><span class="comment">// The cancel function in Tx will be called after done is set to true.</span></span><br><span class="line"> <span class="comment">// 生成一个cancel Contxt，这个context伴随着这个事务，它的生命周期和事务一样长</span></span><br><span class="line">ctx, cancel := context.WithCancel(ctx)</span><br><span class="line">tx = &Tx{</span><br><span class="line">db: db,</span><br><span class="line">dc: dc,</span><br><span class="line">releaseConn: release,</span><br><span class="line">txi: txi,</span><br><span class="line">cancel: cancel,</span><br><span class="line">keepConnOnRollback: keepConnOnRollback,</span><br><span class="line">ctx: ctx,</span><br><span class="line">}</span><br><span class="line"> </span><br><span class="line"> <span class="comment">// 等待事务结束</span></span><br><span class="line"><span class="keyword">go</span> tx.awaitDone()</span><br><span class="line"><span class="keyword">return</span> tx, <span class="literal">nil</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// awaitDone blocks until the context in Tx is canceled and rolls back</span></span><br><span class="line"><span class="comment">// the transaction if it's not already done.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(tx *Tx)</span> <span class="title">awaitDone</span><span class="params">()</span></span> {</span><br><span class="line"><span class="comment">// Wait for either the transaction to be committed or rolled</span></span><br><span class="line"><span class="comment">// back, or for the associated context to be closed.</span></span><br><span class="line"> <span class="comment">// 等待context的结束</span></span><br><span class="line"> <span class="comment">// 在执行tx.Commit()或tx.RollBack()时，会调用cancel()</span></span><br><span class="line"><-tx.ctx.Done()</span><br><span class="line"></span><br><span class="line">discardConnection := !tx.keepConnOnRollback</span><br><span class="line"> <span class="comment">// 如果不是由Commit或RollBack调用的cancel(),导致ctx结束，那么回滚会在这执行</span></span><br><span class="line">tx.rollback(discardConnection)</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="Commit"><a href="#Commit" class="headerlink" title="Commit"></a>Commit</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// Commit commits the transaction.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(tx *Tx)</span> <span class="title">Commit</span><span class="params">()</span> <span class="title">error</span></span> {</span><br><span class="line"> <span class="comment">// Check context first to avoid transaction leak.</span></span><br><span class="line"> <span class="comment">// If put it behind tx.done CompareAndSwap statement, we can't ensure</span></span><br><span class="line"> <span class="comment">// the consistency between tx.done and the real COMMIT operation.</span></span><br><span class="line"> <span class="keyword">select</span> {</span><br><span class="line"> <span class="keyword">default</span>:</span><br><span class="line"> <span class="comment">// 如果没有调用rollback以及ctx没有结束，那么会走到这</span></span><br><span class="line"> <span class="keyword">case</span> <-tx.ctx.Done():</span><br><span class="line"> <span class="keyword">if</span> atomic.LoadInt32(&tx.done) == <span class="number">1</span> {</span><br><span class="line"> <span class="keyword">return</span> ErrTxDone</span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">return</span> tx.ctx.Err()</span><br><span class="line"> }</span><br><span class="line"> </span><br><span class="line"> <span class="comment">// 原子操作,0->1,rollback时也会做这个操作，确保只执行一次</span></span><br><span class="line"> <span class="keyword">if</span> !atomic.CompareAndSwapInt32(&tx.done, <span class="number">0</span>, <span class="number">1</span>) {</span><br><span class="line"> <span class="keyword">return</span> ErrTxDone</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// Cancel the Tx to release any active R-closemu locks.</span></span><br><span class="line"> <span class="comment">// This is safe to do because tx.done has already transitioned</span></span><br><span class="line"> <span class="comment">// from 0 to 1. Hold the W-closemu lock prior to rollback</span></span><br><span class="line"> <span class="comment">// to ensure no other connection has an active query.</span></span><br><span class="line"> tx.cancel() <span class="comment">// 执行cancel方法，结束开始事务时开启的goroutine</span></span><br><span class="line"> tx.closemu.Lock()</span><br><span class="line"> tx.closemu.Unlock()</span><br><span class="line"></span><br><span class="line"> <span class="keyword">var</span> err error</span><br><span class="line"> <span class="comment">// 通过驱动建立的连接执行commit方法</span></span><br><span class="line"> withLock(tx.dc, <span class="function"><span class="keyword">func</span><span class="params">()</span></span> {</span><br><span class="line"> err = tx.txi.Commit()</span><br><span class="line"> })</span><br><span class="line"> <span class="keyword">if</span> !errors.Is(err, driver.ErrBadConn) {</span><br><span class="line"> tx.closePrepared()</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 释放连接 inUse=false</span></span><br><span class="line"> tx.<span class="built_in">close</span>(err)</span><br><span class="line"> <span class="keyword">return</span> err</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="Rollback"><a href="#Rollback" class="headerlink" title="Rollback"></a>Rollback</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// Rollback aborts the transaction.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(tx *Tx)</span> <span class="title">Rollback</span><span class="params">()</span> <span class="title">error</span></span> {</span><br><span class="line"><span class="keyword">return</span> tx.rollback(<span class="literal">false</span>)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// rollback aborts the transaction and optionally forces the pool to discard</span></span><br><span class="line"><span class="comment">// the connection.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(tx *Tx)</span> <span class="title">rollback</span><span class="params">(discardConn <span class="keyword">bool</span>)</span> <span class="title">error</span></span> {</span><br><span class="line"> <span class="comment">// 原子操作 确保只执行一次</span></span><br><span class="line"><span class="keyword">if</span> !atomic.CompareAndSwapInt32(&tx.done, <span class="number">0</span>, <span class="number">1</span>) {</span><br><span class="line"><span class="keyword">return</span> ErrTxDone</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> rollbackHook != <span class="literal">nil</span> {</span><br><span class="line">rollbackHook()</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// Cancel the Tx to release any active R-closemu locks.</span></span><br><span class="line"><span class="comment">// This is safe to do because tx.done has already transitioned</span></span><br><span class="line"><span class="comment">// from 0 to 1. Hold the W-closemu lock prior to rollback</span></span><br><span class="line"><span class="comment">// to ensure no other connection has an active query.</span></span><br><span class="line">tx.cancel()</span><br><span class="line">tx.closemu.Lock()</span><br><span class="line">tx.closemu.Unlock()</span><br><span class="line"></span><br><span class="line"><span class="keyword">var</span> err error</span><br><span class="line"> <span class="comment">// 通过驱动建立的连接执行rollback方法</span></span><br><span class="line">withLock(tx.dc, <span class="function"><span class="keyword">func</span><span class="params">()</span></span> {</span><br><span class="line">err = tx.txi.Rollback()</span><br><span class="line">})</span><br><span class="line"><span class="keyword">if</span> !errors.Is(err, driver.ErrBadConn) {</span><br><span class="line">tx.closePrepared()</span><br><span class="line">}</span><br><span class="line"><span class="keyword">if</span> discardConn {</span><br><span class="line">err = driver.ErrBadConn</span><br><span class="line">}</span><br><span class="line"> <span class="comment">// 释放连接</span></span><br><span class="line">tx.<span class="built_in">close</span>(err)</span><br><span class="line"><span class="keyword">return</span> err</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h3 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h3><p>事务是基于连接的，连接只能被一方使用，在释放之前，无法得到。</p>]]></content>
<summary type="html"><h2 id="Sql源码浅析"><a href="#Sql源码浅析" class="headerlink" title="Sql源码浅析"></a>Sql源码浅析</h2><h3 id="Connect"><a href="#Connect" class="headerlink" title="Connect"></a>Connect</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">Open</span><span class="params">(driverName, dataSourceName <span class="keyword">string</span>)</span> <span class="params">(*DB, error)</span></span> &#123;</span><br><span class="line"> <span class="comment">// 查找已注册的驱动 由_ &quot;github.com/go-sql-driver/mysql&quot;提供</span></span><br><span class="line"> <span class="comment">// 驱动是真正和数据库打交道的</span></span><br><span class="line"> driversMu.RLock()</span><br><span class="line"> driveri, ok := drivers[driverName]</span><br><span class="line"> driversMu.RUnlock()</span><br><span class="line"> <span class="keyword">if</span> !ok &#123;</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>, fmt.Errorf(<span class="string">&quot;sql: unknown driver %q (forgotten import?)&quot;</span>, driverName)</span><br><span class="line"> &#125;</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"></span><br><span class="line"> <span class="keyword">if</span> driverCtx, ok := driveri.(driver.DriverContext); ok &#123;</span><br><span class="line"> <span class="comment">// 获取连接器</span></span><br><span class="line"> connector, err := driverCtx.OpenConnector(dataSourceName)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>, err</span><br><span class="line"> &#125;</span><br><span class="line"> <span class="keyword">return</span> OpenDB(connector), <span class="literal">nil</span></span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> OpenDB(dsnConnector&#123;dsn: dataSourceName, driver: driveri&#125;), <span class="literal">nil</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// OpenDB may just validate its arguments without creating a connection</span></span><br><span class="line"><span class="comment">// to the database. To verify that the data source name is valid, call</span></span><br><span class="line"><span class="comment">// Ping.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">OpenDB</span><span class="params">(c driver.Connector)</span> *<span class="title">DB</span></span> &#123;</span><br><span class="line"> <span class="comment">// 初始化数据，为后续维护连接池做准备</span></span><br><span class="line"> ctx, cancel := context.WithCancel(context.Background())</span><br><span class="line"> db := &amp;DB&#123;</span><br><span class="line"> connector: c,</span><br><span class="line"> openerCh: <span class="built_in">make</span>(<span class="keyword">chan</span> <span class="keyword">struct</span>&#123;&#125;, connectionRequestQueueSize),</span><br><span class="line"> lastPut: <span class="built_in">make</span>(<span class="keyword">map</span>[*driverConn]<span class="keyword">string</span>),</span><br><span class="line"> connRequests: <span class="built_in">make</span>(<span class="keyword">map</span>[<span class="keyword">uint64</span>]<span class="keyword">chan</span> connRequest),</span><br><span class="line"> stop: cancel,</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> <span class="keyword">go</span> db.connectionOpener(ctx)</span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> db</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// Runs in a separate goroutine, opens new connections when requested.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">connectionOpener</span><span class="params">(ctx context.Context)</span></span> &#123;</span><br><span class="line"> <span class="keyword">for</span> &#123;</span><br><span class="line"> <span class="keyword">select</span> &#123;</span><br><span class="line"> <span class="keyword">case</span> &lt;-ctx.Done():</span><br><span class="line"> <span class="keyword">return</span></span><br><span class="line"> <span class="keyword">case</span> &lt;-db.openerCh:</span><br><span class="line"> <span class="comment">// 收到创建新的连接的请求</span></span><br><span class="line"> db.openNewConnection(ctx)</span><br><span class="line"> &#125;</span><br><span class="line"> &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></summary>
<category term="go-sql" scheme="https://cwww3.github.io/tags/go-sql/"/>
</entry>
<entry>
<title>gorm</title>
<link href="https://cwww3.github.io/2022/07/02/gorm/"/>
<id>https://cwww3.github.io/2022/07/02/gorm/</id>
<published>2022-07-02T14:51:13.000Z</published>
<updated>2024-07-04T04:45:40.784Z</updated>
<content type="html"><![CDATA[<h2 id="GORM-源码浅析"><a href="#GORM-源码浅析" class="headerlink" title="GORM 源码浅析"></a>GORM 源码浅析</h2><h3 id="连接"><a href="#连接" class="headerlink" title="连接"></a>连接</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// gorm 连接数据库</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">Open</span><span class="params">(dialector Dialector, opts ...Option)</span> <span class="params">(db *DB, err error)</span></span> {</span><br><span class="line"> <span class="comment">// 全局配置</span></span><br><span class="line"> config := &Config{}</span><br><span class="line"></span><br><span class="line">...</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 加载配置</span></span><br><span class="line"><span class="keyword">for</span> _, opt := <span class="keyword">range</span> opts {</span><br><span class="line"><span class="keyword">if</span> opt != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">if</span> applyErr := opt.Apply(config); applyErr != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span> <span class="literal">nil</span>, applyErr</span><br><span class="line">}</span><br><span class="line"><span class="keyword">defer</span> <span class="function"><span class="keyword">func</span><span class="params">(opt Option)</span></span> {</span><br><span class="line"><span class="keyword">if</span> errr := opt.AfterInitialize(db); errr != <span class="literal">nil</span> {</span><br><span class="line">err = errr</span><br><span class="line">}</span><br><span class="line">}(opt)</span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 时间精度配置 默认毫秒</span></span><br><span class="line"><span class="keyword">if</span> d, ok := dialector.(<span class="keyword">interface</span>{ Apply(*Config) error }); ok {</span><br><span class="line"><span class="keyword">if</span> err = d.Apply(config); err != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span></span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> <span class="comment">// 注册query/create..等操作执行时所需要调用的函数</span></span><br><span class="line"> <span class="comment">// gorm:query -> gorm:preload -> gorm:after_query</span></span><br><span class="line"> <span class="comment">// 对不同版本的数据库做相应的约束配置</span></span><br><span class="line"> <span class="comment">// 调用sql.Open()初始化连接池</span></span><br><span class="line">db.callbacks = initializeCallbacks(db)</span><br><span class="line"><span class="keyword">if</span> config.Dialector != <span class="literal">nil</span> {</span><br><span class="line">err = config.Dialector.Initialize(db)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"> ...</span><br><span class="line"></span><br><span class="line"> <span class="comment">// clone=1</span></span><br><span class="line">db = &DB{Config: config, clone: <span class="number">1</span>}</span><br><span class="line"></span><br><span class="line"><span class="comment">// 在执行sql前后，相关的信息都会存在其中</span></span><br><span class="line">db.Statement = &Statement{</span><br><span class="line">DB: db,</span><br><span class="line">ConnPool: db.ConnPool,</span><br><span class="line">Context: context.Background(),</span><br><span class="line">Clauses: <span class="keyword">map</span>[<span class="keyword">string</span>]clause.Clause{},</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 连接时，自动会进行ping操作</span></span><br><span class="line"><span class="keyword">if</span> err == <span class="literal">nil</span> && !config.DisableAutomaticPing {</span><br><span class="line"><span class="keyword">if</span> pinger, ok := db.ConnPool.(<span class="keyword">interface</span>{ Ping() error }); ok {</span><br><span class="line">err = pinger.Ping()</span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line">config.Logger.Error(context.Background(), <span class="string">"failed to initialize database, got error %v"</span>, err)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span></span><br><span class="line">}</span><br><span class="line"></span><br></pre></td></tr></table></figure><span id="more"></span><h3 id="Session"><a href="#Session" class="headerlink" title="Session"></a>Session</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// Session create new db session 可覆盖全局配置</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">Session</span><span class="params">(config *Session)</span> *<span class="title">DB</span></span> {</span><br><span class="line"><span class="keyword">var</span> (</span><br><span class="line">txConfig = *db.Config</span><br><span class="line">tx = &DB{</span><br><span class="line">Config: &txConfig,</span><br><span class="line">Statement: db.Statement, <span class="comment">// 直接赋值原db的statement</span></span><br><span class="line">Error: db.Error,</span><br><span class="line">clone: <span class="number">1</span>, <span class="comment">// clone = 1</span></span><br><span class="line">}</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 对全局配置进行覆盖</span></span><br><span class="line"><span class="keyword">if</span> config.CreateBatchSize > <span class="number">0</span> {</span><br><span class="line">tx.Config.CreateBatchSize = config.CreateBatchSize</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.SkipDefaultTransaction {</span><br><span class="line">tx.Config.SkipDefaultTransaction = <span class="literal">true</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.AllowGlobalUpdate {</span><br><span class="line">txConfig.AllowGlobalUpdate = <span class="literal">true</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.FullSaveAssociations {</span><br><span class="line">txConfig.FullSaveAssociations = <span class="literal">true</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果条件true 会对原db的statement进行深拷贝(并发安全)</span></span><br><span class="line"> <span class="comment">// 如果不进行clone,原db和新db使用的是同一个statement</span></span><br><span class="line"> <span class="comment">// 在分别执行sql后，信息都会存于同一个statement中，相互影响</span></span><br><span class="line"><span class="keyword">if</span> config.Context != <span class="literal">nil</span> || config.PrepareStmt || config.SkipHooks {</span><br><span class="line">tx.Statement = tx.Statement.clone()</span><br><span class="line">tx.Statement.DB = tx</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.Context != <span class="literal">nil</span> {</span><br><span class="line">tx.Statement.Context = config.Context</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">...</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.DisableNestedTransaction {</span><br><span class="line">txConfig.DisableNestedTransaction = <span class="literal">true</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"> <span class="comment">// newDB=true clone=1</span></span><br><span class="line"> <span class="comment">// newDB=false clone=2</span></span><br><span class="line"> <span class="comment">// clone值的影响在getInstance()中体现</span></span><br><span class="line"><span class="keyword">if</span> !config.NewDB {</span><br><span class="line">tx.clone = <span class="number">2</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.DryRun {</span><br><span class="line">tx.Config.DryRun = <span class="literal">true</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.QueryFields {</span><br><span class="line">tx.Config.QueryFields = <span class="literal">true</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.Logger != <span class="literal">nil</span> {</span><br><span class="line">tx.Config.Logger = config.Logger</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.NowFunc != <span class="literal">nil</span> {</span><br><span class="line">tx.Config.NowFunc = config.NowFunc</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> config.Initialized {</span><br><span class="line">tx = tx.getInstance()</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span> tx</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure><h3 id="getInstance"><a href="#getInstance" class="headerlink" title="getInstance"></a>getInstance</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="comment">// 在执行大部分语句之前都会调用该方法</span></span><br><span class="line"><span class="comment">// Model() Where() Order() Find() ...</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">getInstance</span><span class="params">()</span> *<span class="title">DB</span></span> {</span><br><span class="line"> <span class="comment">// 根据clone的值，做相应的处理</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果clone=1，会将statement清空，原先设置的条件(比如where order...)都会清除</span></span><br><span class="line"> <span class="comment">// 如果clone=2, 会将statement进行深拷贝，原先的条件保留，但是新老db互不影响()</span></span><br><span class="line"> <span class="comment">// 如果clone=0，直接返回，不做任何操作</span></span><br><span class="line"> <span class="comment">// 做一些build(拼接)操作，比如执行Where(),Order()等操作时，直接使用原db即可，不需要对statement进行拷贝或清除</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> db.clone > <span class="number">0</span> {</span><br><span class="line">tx := &DB{Config: db.Config, Error: db.Error}</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> db.clone == <span class="number">1</span> {</span><br><span class="line"><span class="comment">// clone with new statement</span></span><br><span class="line">tx.Statement = &Statement{</span><br><span class="line">DB: tx,</span><br><span class="line">ConnPool: db.Statement.ConnPool,</span><br><span class="line">Context: db.Statement.Context,</span><br><span class="line">Clauses: <span class="keyword">map</span>[<span class="keyword">string</span>]clause.Clause{},</span><br><span class="line">Vars: <span class="built_in">make</span>([]<span class="keyword">interface</span>{}, <span class="number">0</span>, <span class="number">8</span>),</span><br><span class="line">}</span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line"><span class="comment">// with clone statement</span></span><br><span class="line">tx.Statement = db.Statement.clone()</span><br><span class="line">tx.Statement.DB = tx</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span> tx</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span> db</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h3 id="Tx"><a href="#Tx" class="headerlink" title="Tx"></a>Tx</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// Transaction start a transaction as a block, return error will rollback, otherwise to commit.</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(db *DB)</span> <span class="title">Transaction</span><span class="params">(fc <span class="keyword">func</span>(tx *DB)</span> <span class="title">error</span>, <span class="title">opts</span> ...*<span class="title">sql</span>.<span class="title">TxOptions</span>) <span class="params">(err error)</span></span> {</span><br><span class="line">panicked := <span class="literal">true</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> committer, ok := db.Statement.ConnPool.(TxCommitter); ok && committer != <span class="literal">nil</span> {</span><br><span class="line"><span class="comment">// nested transaction</span></span><br><span class="line"> <span class="comment">// 嵌套事务会进入该分支，原理是一个事务可以分为多个段，用户可以将事务回滚到指定的段，而不是整个事务</span></span><br><span class="line"> <span class="comment">// https://dotnettutorials.net/lesson/savepoint-in-mysql/</span></span><br><span class="line"><span class="keyword">if</span> !db.DisableNestedTransaction {</span><br><span class="line">err = db.SavePoint(fmt.Sprintf(<span class="string">"sp%p"</span>, fc)).Error</span><br><span class="line"><span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">defer</span> <span class="function"><span class="keyword">func</span><span class="params">()</span></span> {</span><br><span class="line"><span class="comment">// Make sure to rollback when panic, Block error or Commit error</span></span><br><span class="line"><span class="keyword">if</span> panicked || err != <span class="literal">nil</span> {</span><br><span class="line">db.RollbackTo(fmt.Sprintf(<span class="string">"sp%p"</span>, fc))</span><br><span class="line">}</span><br><span class="line">}()</span><br><span class="line">}</span><br><span class="line">err = fc(db.Session(&Session{NewDB: db.clone == <span class="number">1</span>}))</span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line"> <span class="comment">// 非嵌套事务 开启事务</span></span><br><span class="line">tx := db.Begin(opts...)</span><br><span class="line"><span class="keyword">if</span> tx.Error != <span class="literal">nil</span> {</span><br><span class="line"><span class="keyword">return</span> tx.Error</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">defer</span> <span class="function"><span class="keyword">func</span><span class="params">()</span></span> {</span><br><span class="line"><span class="comment">// Make sure to rollback when panic, Block error or Commit error</span></span><br><span class="line"><span class="keyword">if</span> panicked || err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="comment">// rollback 事务</span></span><br><span class="line">tx.Rollback()</span><br><span class="line">}</span><br><span class="line">}()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> err = fc(tx); err == <span class="literal">nil</span> {</span><br><span class="line">panicked = <span class="literal">false</span></span><br><span class="line"> <span class="comment">// commit 事务</span></span><br><span class="line"><span class="keyword">return</span> tx.Commit().Error</span><br><span class="line">}</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">panicked = <span class="literal">false</span></span><br><span class="line"><span class="keyword">return</span></span><br><span class="line">}</span><br><span class="line"></span><br></pre></td></tr></table></figure><h4 id="嵌套事务"><a href="#嵌套事务" class="headerlink" title="嵌套事务"></a>嵌套事务</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">db.Transaction(<span class="function"><span class="keyword">func</span><span class="params">(tx *gorm.DB)</span> <span class="title">error</span></span> {</span><br><span class="line"> tx.Transaction(<span class="function"><span class="keyword">func</span><span class="params">(tx2 *gorm.DB)</span> <span class="title">error</span></span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span></span><br><span class="line"> })</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span></span><br><span class="line">})</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><h2 id="GORM-源码浅析"><a href="#GORM-源码浅析" class="headerlink" title="GORM 源码浅析"></a>GORM 源码浅析</h2><h3 id="连接"><a href="#连接" class="headerlink" title="连接"></a>连接</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// gorm 连接数据库</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">Open</span><span class="params">(dialector Dialector, opts ...Option)</span> <span class="params">(db *DB, err error)</span></span> &#123;</span><br><span class="line"> <span class="comment">// 全局配置</span></span><br><span class="line"> config := &amp;Config&#123;&#125;</span><br><span class="line"></span><br><span class="line"> ...</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 加载配置</span></span><br><span class="line"> <span class="keyword">for</span> _, opt := <span class="keyword">range</span> opts &#123;</span><br><span class="line"> <span class="keyword">if</span> opt != <span class="literal">nil</span> &#123;</span><br><span class="line"> <span class="keyword">if</span> applyErr := opt.Apply(config); applyErr != <span class="literal">nil</span> &#123;</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>, applyErr</span><br><span class="line"> &#125;</span><br><span class="line"> <span class="keyword">defer</span> <span class="function"><span class="keyword">func</span><span class="params">(opt Option)</span></span> &#123;</span><br><span class="line"> <span class="keyword">if</span> errr := opt.AfterInitialize(db); errr != <span class="literal">nil</span> &#123;</span><br><span class="line"> err = errr</span><br><span class="line"> &#125;</span><br><span class="line"> &#125;(opt)</span><br><span class="line"> &#125;</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 时间精度配置 默认毫秒</span></span><br><span class="line"> <span class="keyword">if</span> d, ok := dialector.(<span class="keyword">interface</span>&#123; Apply(*Config) error &#125;); ok &#123;</span><br><span class="line"> <span class="keyword">if</span> err = d.Apply(config); err != <span class="literal">nil</span> &#123;</span><br><span class="line"> <span class="keyword">return</span></span><br><span class="line"> &#125;</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> <span class="comment">// 注册query/create..等操作执行时所需要调用的函数</span></span><br><span class="line"> <span class="comment">// gorm:query -&gt; gorm:preload -&gt; gorm:after_query</span></span><br><span class="line"> <span class="comment">// 对不同版本的数据库做相应的约束配置</span></span><br><span class="line"> <span class="comment">// 调用sql.Open()初始化连接池</span></span><br><span class="line"> db.callbacks = initializeCallbacks(db)</span><br><span class="line"> <span class="keyword">if</span> config.Dialector != <span class="literal">nil</span> &#123;</span><br><span class="line"> err = config.Dialector.Initialize(db)</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> ...</span><br><span class="line"></span><br><span class="line"> <span class="comment">// clone=1</span></span><br><span class="line"> db = &amp;DB&#123;Config: config, clone: <span class="number">1</span>&#125;</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 在执行sql前后，相关的信息都会存在其中</span></span><br><span class="line"> db.Statement = &amp;Statement&#123;</span><br><span class="line"> DB: db,</span><br><span class="line"> ConnPool: db.ConnPool,</span><br><span class="line"> Context: context.Background(),</span><br><span class="line"> Clauses: <span class="keyword">map</span>[<span class="keyword">string</span>]clause.Clause&#123;&#125;,</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 连接时，自动会进行ping操作</span></span><br><span class="line"> <span class="keyword">if</span> err == <span class="literal">nil</span> &amp;&amp; !config.DisableAutomaticPing &#123;</span><br><span class="line"> <span class="keyword">if</span> pinger, ok := db.ConnPool.(<span class="keyword">interface</span>&#123; Ping() error &#125;); ok &#123;</span><br><span class="line"> err = pinger.Ping()</span><br><span class="line"> &#125;</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> &#123;</span><br><span class="line"> config.Logger.Error(context.Background(), <span class="string">&quot;failed to initialize database, got error %v&quot;</span>, err)</span><br><span class="line"> &#125;</span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure></summary>
<category term="Gorm" scheme="https://cwww3.github.io/tags/Gorm/"/>
</entry>
<entry>
<title>mysqlbinlog</title>
<link href="https://cwww3.github.io/2022/03/31/mysqlbinlog/"/>
<id>https://cwww3.github.io/2022/03/31/mysqlbinlog/</id>
<published>2022-03-31T01:19:54.000Z</published>
<updated>2022-03-31T01:20:26.430Z</updated>
<content type="html"><![CDATA[<p>查看是否开启binlog</p><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">show variables like 'log_bin';</span><br></pre></td></tr></table></figure><p>查看binlog</p><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">show binary logs</span><br></pre></td></tr></table></figure><span id="more"></span><p>分析binlog</p><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">mysqlbinlog </span><br><span class="line">-base64-output=decode-rows -- 解码</span><br><span class="line">--no-defaults --database=db </span><br><span class="line">--start-datetime='2019-04-11 00:00:00' </span><br><span class="line">--stop-datetime='2019-04-11 15:00:00' mysql-bin.000007 </span><br><span class="line">-- 指定位置 --start-position --stop-position</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><p>查看是否开启binlog</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">show variables like &#x27;log_bin&#x27;;</span><br></pre></td></tr></table></figure>
<p>查看binlog</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">show binary logs</span><br></pre></td></tr></table></figure></summary>
<category term="MySql" scheme="https://cwww3.github.io/tags/MySql/"/>
</entry>
<entry>
<title>map</title>
<link href="https://cwww3.github.io/2022/01/06/map/"/>
<id>https://cwww3.github.io/2022/01/06/map/</id>
<published>2022-01-05T16:29:49.000Z</published>
<updated>2023-05-11T04:46:36.491Z</updated>
<content type="html"><![CDATA[<h2 id="Map"><a href="#Map" class="headerlink" title="Map"></a>Map</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// A header for a Go map.</span></span><br><span class="line"><span class="keyword">type</span> hmap <span class="keyword">struct</span> {</span><br><span class="line"> <span class="comment">// 元素个数，调用 len(map) 时，直接返回此值</span></span><br><span class="line"> count <span class="keyword">int</span></span><br><span class="line"> flags <span class="keyword">uint8</span></span><br><span class="line"> <span class="comment">// buckets 的对数</span></span><br><span class="line"> B <span class="keyword">uint8</span></span><br><span class="line"> <span class="comment">// overflow 的 bucket 近似数</span></span><br><span class="line"> noverflow <span class="keyword">uint16</span></span><br><span class="line"> <span class="comment">// 计算 key 的哈希的时候会传入哈希函数</span></span><br><span class="line"> hash0 <span class="keyword">uint32</span></span><br><span class="line"> <span class="comment">// 指向 buckets 数组，大小为 2^B</span></span><br><span class="line"> <span class="comment">// 如果元素个数为0，就为 nil</span></span><br><span class="line"> buckets unsafe.Pointer</span><br><span class="line"> <span class="comment">// 扩容的时候，buckets 长度会是 oldbuckets 的两倍</span></span><br><span class="line"> oldbuckets unsafe.Pointer</span><br><span class="line"> <span class="comment">// 指示扩容进度，小于此地址的 buckets 迁移完成</span></span><br><span class="line"> nevacuate <span class="keyword">uintptr</span></span><br><span class="line"> extra *mapextra <span class="comment">// optional fields</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><span id="more"></span><p>buckets 是一个指针，最终它指向的是一个结构体：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> bmap <span class="keyword">struct</span> {</span><br><span class="line">tophash [bucketCnt]<span class="keyword">uint8</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>编译期间会给它加料，动态地创建一个新的结构：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> bmap <span class="keyword">struct</span> {</span><br><span class="line"> topbits [<span class="number">8</span>]<span class="keyword">uint8</span></span><br><span class="line"> keys [<span class="number">8</span>]keytype</span><br><span class="line"> values [<span class="number">8</span>]valuetype</span><br><span class="line"> pad <span class="keyword">uintptr</span></span><br><span class="line"> overflow <span class="keyword">uintptr</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p><code>bmap</code> 就是我们常说的“桶”，桶里面会最多装 8 个 key，这些 key 之所以会落入同一个桶，是因为它们经过哈希计算后，哈希结果是“一类”的(低位一样)。在桶内，又会根据 key 计算出来的 hash 值的高 8 位来决定 key 到底落入桶内的哪个位置（一个桶内最多有 8 个位置）。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104233340464.png" alt="image-20220104233340464"></p><p>bmap 是存放 k-v 的地方，我们把视角拉近，仔细看 bmap 的内部组成。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104233831611.png" alt="image-20220104233831611" style="zoom:50%;" /><p>上图就是 bucket 的内存模型，<code>HOB Hash</code> 指的就是 top hash。 注意到 key 和 value 是各自放在一起的，并不是 <code>key/value/key/value/...</code> 这样的形式。这样的好处是在某些情况下可以省略掉 padding 字段，节省内存空间。</p><h3 id="创建-map"><a href="#创建-map" class="headerlink" title="创建 map"></a>创建 map</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">makemap</span><span class="params">(t *maptype, hint <span class="keyword">int64</span>, h *hmap, bucket unsafe.Pointer)</span> *<span class="title">hmap</span></span> {</span><br><span class="line"> <span class="comment">// 省略各种条件检查...</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// 找到一个 B，使得 map 的装载因子在正常范围内</span></span><br><span class="line"> B := <span class="keyword">uint8</span>(<span class="number">0</span>)</span><br><span class="line"> <span class="keyword">for</span> ; overLoadFactor(hint, B); B++ {</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 初始化 hash table</span></span><br><span class="line"> <span class="comment">// 如果 B 等于 0，那么 buckets 就会在赋值的时候再分配</span></span><br><span class="line"> <span class="comment">// 如果长度比较大，分配内存会花费长一点</span></span><br><span class="line"> buckets := bucket</span><br><span class="line"> <span class="keyword">var</span> extra *mapextra</span><br><span class="line"> <span class="keyword">if</span> B != <span class="number">0</span> {</span><br><span class="line"> <span class="keyword">var</span> nextOverflow *bmap</span><br><span class="line"> buckets, nextOverflow = makeBucketArray(t, B)</span><br><span class="line"> <span class="keyword">if</span> nextOverflow != <span class="literal">nil</span> {</span><br><span class="line"> extra = <span class="built_in">new</span>(mapextra)</span><br><span class="line"> extra.nextOverflow = nextOverflow</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 初始化 hamp</span></span><br><span class="line"> <span class="keyword">if</span> h == <span class="literal">nil</span> {</span><br><span class="line"> h = (*hmap)(newobject(t.hmap))</span><br><span class="line"> }</span><br><span class="line"> h.count = <span class="number">0</span></span><br><span class="line"> h.B = B</span><br><span class="line"> h.extra = extra</span><br><span class="line"> h.flags = <span class="number">0</span></span><br><span class="line"> h.hash0 = fastrand()</span><br><span class="line"> h.buckets = buckets</span><br><span class="line"> h.oldbuckets = <span class="literal">nil</span></span><br><span class="line"> h.nevacuate = <span class="number">0</span></span><br><span class="line"> h.noverflow = <span class="number">0</span></span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> h</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>注意，这个函数返回的结果：<code>*hmap</code>，它是一个指针，而之前讲过的 <code>makeslice</code> 函数返回的是 <code>slice</code> 结构体</p><h3 id="key-定位过程"><a href="#key-定位过程" class="headerlink" title="key 定位过程"></a>key 定位过程</h3><p>key 经过哈希计算后得到哈希值，共 64 个 bit 位（64 位机，32 位机就不讨论了，现在主流都是 64 位机），计算它<strong>到底要落在哪个桶时，只会用到最后 B 个 bit 位</strong>。还记得前面提到过的 B 吗？如果 B = 5，那么桶的数量，也就是 buckets 数组的长度是 2^5 = 32。</p><p>例如，现在有一个 key 经过哈希函数计算后，得到的哈希结果是：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">10010111</span> | <span class="number">000011110110110010001111001010100010010110010101010</span> │ <span class="number">01010</span></span><br></pre></td></tr></table></figure><p>用最后的 5 个 bit 位，也就是 <code>01010</code>，值为 10，也就是 10 号桶。这个操作实际上就是取余操作，但是取余开销太大，所以代码实现上用的位操作代替。</p><p><strong>再用哈希值的高 8 位，找到此 key 在 bucket 中的位置</strong> (先找到 tophash 的位置，再通过计算得到 k,v 的位置)，这是在寻找已有的 key。最开始桶内还没有 key，新加入的 key 会找到第一个空位，放入。</p><p>buckets 编号就是桶编号，当两个不同的 key 落在同一个桶中，也就是发生了哈希冲突。冲突的解决手段是用链表法：在 bucket 中，从前往后找到第一个空位。这样，在查找某个 key 时，先找到对应的桶，再去遍历 bucket 中的 key。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104234423872.png" alt="image-20220104234423872" style="zoom:50%;" /><p>上图中，假定 B = 5，所以 bucket 总数就是 2^5 = 32。首先计算出待查找 key 的哈希，使用低 5 位 <code>00110</code>，找到对应的 6 号 bucket，使用高 8 位 <code>10010111</code>，对应十进制 151，在 6 号 bucket 中寻找 tophash 值（HOB hash）为 151 的 key，找到了 2 号槽位，这样整个查找过程就结束了。</p><p>如果在 bucket 中没找到，并且 overflow 不为空，还要继续去 overflow bucket 中寻找，直到找到或是所有的 key 槽位都找遍了，包括所有的 overflow bucket。</p><p>查找某个 key 的底层函数是 <code>mapacess</code> 系列函数</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">mapaccess1</span><span class="params">(t *maptype, h *hmap, key unsafe.Pointer)</span> <span class="title">unsafe</span>.<span class="title">Pointer</span></span> {</span><br><span class="line"> <span class="comment">// ……</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果 h 什么都没有，返回零值</span></span><br><span class="line"> <span class="keyword">if</span> h == <span class="literal">nil</span> || h.count == <span class="number">0</span> {</span><br><span class="line"> <span class="keyword">return</span> unsafe.Pointer(&zeroVal[<span class="number">0</span>])</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 写和读冲突</span></span><br><span class="line"> <span class="keyword">if</span> h.flags&hashWriting != <span class="number">0</span> {</span><br><span class="line"> throw(<span class="string">"concurrent map read and map write"</span>)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 不同类型 key 使用的 hash 算法在编译期确定</span></span><br><span class="line"> alg := t.key.alg</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 计算哈希值，并且加入 hash0 引入随机性</span></span><br><span class="line"> hash := alg.hash(key, <span class="keyword">uintptr</span>(h.hash0))</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 比如 B=5，那 m 就是31，二进制是全 1</span></span><br><span class="line"> <span class="comment">// 求 bucket num 时，将 hash 与 m 相与，</span></span><br><span class="line"> <span class="comment">// 达到 bucket num 由 hash 的低 8 位决定的效果</span></span><br><span class="line"> m := <span class="keyword">uintptr</span>(<span class="number">1</span>)<<h.B - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// b 就是 bucket 的地址</span></span><br><span class="line"> b := (*bmap)(add(h.buckets, (hash&m)*<span class="keyword">uintptr</span>(t.bucketsize)))</span><br><span class="line"></span><br><span class="line"> <span class="comment">// oldbuckets 不为 nil，说明发生了扩容</span></span><br><span class="line"> <span class="keyword">if</span> c := h.oldbuckets; c != <span class="literal">nil</span> {</span><br><span class="line"> <span class="comment">// 如果不是同 size 扩容（看后面扩容的内容）</span></span><br><span class="line"> <span class="comment">// 对应条件 1 的解决方案</span></span><br><span class="line"> <span class="keyword">if</span> !h.sameSizeGrow() {</span><br><span class="line"> <span class="comment">// 新 bucket 数量是老的 2 倍</span></span><br><span class="line"> m >>= <span class="number">1</span></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 求出 key 在老的 map 中的 bucket 位置</span></span><br><span class="line"> oldb := (*bmap)(add(c, (hash&m)*<span class="keyword">uintptr</span>(t.bucketsize)))</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果 oldb 没有搬迁到新的 bucket</span></span><br><span class="line"> <span class="comment">// 那就在老的 bucket 中寻找</span></span><br><span class="line"> <span class="keyword">if</span> !evacuated(oldb) {</span><br><span class="line"> b = oldb</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 计算出高 8 位的 hash</span></span><br><span class="line"> <span class="comment">// 相当于右移 56 位，只取高8位</span></span><br><span class="line"> top := <span class="keyword">uint8</span>(hash >> (sys.PtrSize*<span class="number">8</span> - <span class="number">8</span>))</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 增加一个 minTopHash</span></span><br><span class="line"> <span class="keyword">if</span> top < minTopHash {</span><br><span class="line"> top += minTopHash</span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">for</span> {</span><br><span class="line"> <span class="comment">// 遍历 8 个 bucket</span></span><br><span class="line"> <span class="keyword">for</span> i := <span class="keyword">uintptr</span>(<span class="number">0</span>); i < bucketCnt; i++ {</span><br><span class="line"> <span class="comment">// tophash 不匹配，继续</span></span><br><span class="line"> <span class="keyword">if</span> b.tophash[i] != top {</span><br><span class="line"> <span class="keyword">continue</span></span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// tophash 匹配，定位到 key 的位置</span></span><br><span class="line"> k := add(unsafe.Pointer(b), dataOffset+i*<span class="keyword">uintptr</span>(t.keysize))</span><br><span class="line"> <span class="comment">// key 是指针</span></span><br><span class="line"> <span class="keyword">if</span> t.indirectkey {</span><br><span class="line"> <span class="comment">// 解引用</span></span><br><span class="line"> k = *((*unsafe.Pointer)(k))</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 如果 key 相等</span></span><br><span class="line"> <span class="keyword">if</span> alg.equal(key, k) {</span><br><span class="line"> <span class="comment">// 定位到 value 的位置</span></span><br><span class="line"> v := add(unsafe.Pointer(b), dataOffset+bucketCnt*<span class="keyword">uintptr</span>(t.keysize)+i*<span class="keyword">uintptr</span>(t.valuesize))</span><br><span class="line"> <span class="comment">// value 解引用</span></span><br><span class="line"> <span class="keyword">if</span> t.indirectvalue {</span><br><span class="line"> v = *((*unsafe.Pointer)(v))</span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">return</span> v</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// bucket 找完（还没找到），继续到 overflow bucket 里找</span></span><br><span class="line"> b = b.overflow(t)</span><br><span class="line"> <span class="comment">// overflow bucket 也找完了，说明没有目标 key</span></span><br><span class="line"> <span class="comment">// 返回零值</span></span><br><span class="line"> <span class="keyword">if</span> b == <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">return</span> unsafe.Pointer(&zeroVal[<span class="number">0</span>])</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>b 是 bmap 的地址，这里 bmap 还是源码里定义的结构体，只包含一个 tophash 数组，经编译器扩充之后的结构体才包含 key，value，overflow 这些字段。dataOffset 是 key 相对于 bmap 起始地址的偏移：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">dataOffset = unsafe.Offsetof(<span class="keyword">struct</span> {</span><br><span class="line"> b bmap</span><br><span class="line"> v <span class="keyword">int64</span></span><br><span class="line"> }{}.v)</span><br></pre></td></tr></table></figure><p>再说整个大循环的写法，最外层是一个无限循环，通过</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">b = b.overflow(t)</span><br></pre></td></tr></table></figure><p>遍历所有的 bucket，这相当于是一个 bucket 链表。</p><p>当定位到一个具体的 bucket 时，里层循环就是遍历这个 bucket 里所有的 cell，或者说所有的槽位，也就是 bucketCnt=8 个槽位。整个循环过程：</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104235126819.png" alt="image-20220104235126819"></p><p>再说一下 <strong>minTopHash</strong>，当一个 cell 的 tophash 值小于 minTopHash 时，标志这个 cell 的迁移状态。因为这个状态值是放在 tophash 数组里，<strong>为了和正常的哈希值区分开，会给 key 计算出来的哈希值一个增量</strong>：minTopHash。这样就能区分正常的 top hash 值和表示状态的哈希值。（如果算出来的哈希值在 0-3 之间，那么给他加一个 minTopHash，前面 0-3 仅用来表示状态。）</p><p>下面的这几种状态就表征了 bucket 的情况：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 空的 cell，也是初始时 bucket 的状态</span></span><br><span class="line">empty = <span class="number">0</span></span><br><span class="line"><span class="comment">// 空的 cell，表示 cell 已经被迁移到新的 bucket</span></span><br><span class="line">evacuatedEmpty = <span class="number">1</span></span><br><span class="line"><span class="comment">// key,value 已经搬迁完毕，但是 key 都在新 bucket 前半部分，</span></span><br><span class="line"><span class="comment">// 后面扩容部分会再讲到。</span></span><br><span class="line">evacuatedX = <span class="number">2</span></span><br><span class="line"><span class="comment">// 同上，key 在后半部分</span></span><br><span class="line">evacuatedY = <span class="number">3</span></span><br><span class="line"><span class="comment">// tophash 的最小正常值</span></span><br><span class="line">minTopHash = <span class="number">4</span></span><br></pre></td></tr></table></figure><p>源码里判断这个 bucket 是否已经搬迁完毕，用到的函数：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">evacuated</span><span class="params">(b *bmap)</span> <span class="title">bool</span></span> {</span><br><span class="line"> h := b.tophash[<span class="number">0</span>]</span><br><span class="line"> <span class="keyword">return</span> h > empty && h < minTopHash</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h3 id="删除"><a href="#删除" class="headerlink" title="删除"></a>删除</h3><p>底层的执行函数是 <code>mapdelete</code></p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">mapdelete</span><span class="params">(t *maptype, h *hmap, key unsafe.Pointer)</span></span></span><br></pre></td></tr></table></figure><p>它首先会检查 h.flags 标志，如果发现写标位是 1，直接 panic，因为这表明有其他协程同时在进行写操作。</p><p>计算 key 的哈希，找到落入的 bucket。检查此 map 如果正在扩容的过程中，直接触发一次搬迁操作。</p><p>删除操作同样是两层循环，核心还是找到 key 的具体位置。寻找过程都是类似的，在 bucket 中挨个 cell 寻找。</p><p>找到对应位置后，对 key 或者 value 进行“清零”操作：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 对 key 清零</span></span><br><span class="line"><span class="keyword">if</span> t.indirectkey {</span><br><span class="line"> *(*unsafe.Pointer)(k) = <span class="literal">nil</span></span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line"> typedmemclr(t.key, k)</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// 对 value 清零</span></span><br><span class="line"><span class="keyword">if</span> t.indirectvalue {</span><br><span class="line"> *(*unsafe.Pointer)(v) = <span class="literal">nil</span></span><br><span class="line">} <span class="keyword">else</span> {</span><br><span class="line"> typedmemclr(t.elem, v)</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>最后，将 count 值减 1，将对应位置的 tophash 值置成 <code>Empty</code>。</p><h3 id="遍历"><a href="#遍历" class="headerlink" title="遍历"></a>遍历</h3><p>本来 map 的遍历过程比较简单：遍历所有的 bucket 以及它后面挂的 overflow bucket，然后挨个遍历 bucket 中的所有 cell。每个 bucket 中包含 8 个 cell，从有 key 的 cell 中取出 key 和 value，这个过程就完成了。</p><p>但是，现实并没有这么简单。还记得前面讲过的扩容过程吗？扩容过程不是一个原子的操作，它每次最多只搬运 2 个 bucket，所以如果触发了扩容操作，那么在很长时间里，map 的状态都是处于一个中间态：有些 bucket 已经搬迁到新家，而有些 bucket 还待在老地方。</p><p>因此，遍历如果发生在扩容的过程中，就会涉及到遍历新老 bucket 的过程，这是难点所在。</p><p>先是调用 <code>mapiterinit</code> 函数初始化迭代器，然后循环调用 <code>mapiternext</code> 函数进行 map 迭代。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220105000801438.png" alt="image-20220105000801438" style="zoom:50%;" /><p>即使是对一个写死的 map 进行遍历，每次出来的结果也是无序的，近距离地观察他们的实现。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 生成随机数 r</span></span><br><span class="line">r := <span class="keyword">uintptr</span>(fastrand())</span><br><span class="line"><span class="keyword">if</span> h.B > <span class="number">31</span>-bucketCntBits {</span><br><span class="line"> r += <span class="keyword">uintptr</span>(fastrand()) << <span class="number">31</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// 从哪个 bucket 开始遍历</span></span><br><span class="line">it.startBucket = r & (<span class="keyword">uintptr</span>(<span class="number">1</span>)<<h.B - <span class="number">1</span>)</span><br><span class="line"><span class="comment">// 从 bucket 的哪个 cell 开始遍历</span></span><br><span class="line">it.offset = <span class="keyword">uint8</span>(r >> h.B & (bucketCnt - <span class="number">1</span>))</span><br></pre></td></tr></table></figure><p>假设我们有下图所示的一个 map，起始时 B = 1，有两个 bucket，后来触发了扩容（这里不要深究扩容条件，只是一个设定），B 变成 2。并且， 1 号 bucket 中的内容搬迁到了新的 bucket，<code>1 号</code>裂变成 <code>1 号</code>和 <code>3 号</code>；<code>0 号</code> bucket 暂未搬迁。老的 bucket 挂在在 <code>*oldbuckets</code> 指针上面，新的 bucket 则挂在 <code>*buckets</code> 指针上面。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220105000418534.png" alt="image-20220105000418534"></p><p>这时，我们对此 map 进行遍历。假设经过初始化后，startBucket = 3，offset = 2。于是，遍历的起点将是 3 号 bucket 的 2 号 cell，下面这张图就是开始遍历时的状态：</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220105000448649.png" alt="image-20220105000448649"></p><p>标红的表示起始位置，bucket 遍历顺序为：3 -> 0 -> 1 -> 2。因为 3 号 bucket 对应老的 1 号 bucket，因此先检查老 1 号 bucket 是否已经被搬迁过。</p><p>在本例中，老 1 号 bucket 已经被搬迁过了。所以它的 tophash[0] 值在 (0,4) 范围内，因此只用遍历新的 3 号 bucket。依次遍历 3 号 bucket 的 cell，这时候会找到第一个非空的 key：元素 e。到这里，mapiternext 函数返回，这时我们的遍历结果仅有一个元素：由于返回的 key 不为空，所以会继续调用 mapiternext 函数。继续从上次遍历到的地方往后遍历，从新 3 号 overflow bucket 中找到了元素 f 和 元素 g。</p><p>新 3 号 bucket 遍历完之后，回到了新 0 号 bucket。0 号 bucket 对应老的 0 号 bucket，经检查，老 0 号 bucket 并未搬迁，因此对新 0 号 bucket 的遍历就改为遍历老 0 号 bucket。<strong>那是不是把老 0 号 bucket 中的所有 key 都取出来呢？</strong></p><p>并没有这么简单，回忆一下，老 0 号 bucket 在搬迁后将裂变成 2 个 bucket：新 0 号、新 2 号。而我们此时正在遍历的只是新 0 号 bucket（注意，遍历都是遍历的 <code>*bucket</code> 指针，也就是所谓的新 buckets）。所以，我们<strong>只会取出老 0 号 bucket 中那些在裂变之后，分配到新 0 号 bucket 中的那些 key</strong>。</p><p>因此，<code>lowbits == 00</code> 的将进入遍历结果集：和之前的流程一样，继续遍历新 1 号 bucket，发现老 1 号 bucket 已经搬迁，只用遍历新 1 号 bucket 中现有的元素就可以了。</p><p>继续遍历新 2 号 bucket，它来自老 0 号 bucket，因此需要在老 0 号 bucket 中那些会裂变到新 2 号 bucket 中的 key，也就是 <code>lowbit == 10</code> 的那些 key。</p><p>最后，继续遍历到新 3 号 bucket 时，发现所有的 bucket 都已经遍历完毕，整个迭代过程执行完毕。</p><h3 id="扩容"><a href="#扩容" class="headerlink" title="扩容"></a>扩容</h3><p>使用哈希表的目的就是要快速查找到目标 key，然而，随着向 map 中添加的 key 越来越多，key 发生碰撞的概率也越来越大。bucket 中的 8 个 cell 会被逐渐塞满，查找、插入、删除 key 的效率也会越来越低。最理想的情况是一个 bucket 只装一个 key，这样，就能达到 <code>O(1)</code> 的效率，但这样空间消耗太大，用空间换时间的代价太高。</p><p>Go 语言采用一个 bucket 里装载 8 个 key，定位到某个 bucket 后，还需要再定位到具体的 key，这实际上又用了时间换空间。</p><p>当然，这样做，要有一个度，不然所有的 key 都落在了同一个 bucket 里，直接退化成了链表，各种操作的效率直接降为 O(n)，是不行的。</p><p>因此，需要有一个指标来衡量前面描述的情况，这就是<code>装载因子</code>。Go 源码里这样定义 <code>装载因子</code>：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">loadFactor := count / (<span class="number">2</span>^B)</span><br></pre></td></tr></table></figure><p>count 就是 map 的元素个数，2^B 表示 bucket 数量。</p><p>再来说触发 map 扩容的时机：在向 map 插入新 key 的时候，会进行条件检测，符合下面这 2 个条件，就会触发扩容：</p><ol><li>装载因子超过阈值，源码里定义的阈值是 6.5。</li><li>overflow 的 bucket 数量过多：当 B 小于 15，也就是 bucket 总数 2^B 小于 2^15 时，如果 overflow 的 bucket 数量超过 2^B；当 B >= 15，也就是 bucket 总数 2^B 大于等于 2^15，如果 overflow 的 bucket 数量超过 2^15。</li></ol><p>通过汇编语言可以找到赋值操作对应源码中的函数是 <code>mapassign</code>，对应扩容条件的源码如下：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// src/runtime/hashmap.go/mapassign</span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 触发扩容时机</span></span><br><span class="line"><span class="keyword">if</span> !h.growing() && (overLoadFactor(<span class="keyword">int64</span>(h.count), h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) {</span><br><span class="line"> hashGrow(t, h)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"><span class="comment">// 装载因子超过 6.5</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">overLoadFactor</span><span class="params">(count <span class="keyword">int64</span>, B <span class="keyword">uint8</span>)</span> <span class="title">bool</span></span> {</span><br><span class="line"> <span class="keyword">return</span> count >= bucketCnt && <span class="keyword">float32</span>(count) >= loadFactor*<span class="keyword">float32</span>((<span class="keyword">uint64</span>(<span class="number">1</span>)<<B))</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// overflow buckets 太多</span></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">tooManyOverflowBuckets</span><span class="params">(noverflow <span class="keyword">uint16</span>, B <span class="keyword">uint8</span>)</span> <span class="title">bool</span></span> {</span><br><span class="line"> <span class="keyword">if</span> B < <span class="number">16</span> {</span><br><span class="line"> <span class="keyword">return</span> noverflow >= <span class="keyword">uint16</span>(<span class="number">1</span>)<<B</span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">return</span> noverflow >= <span class="number">1</span><<<span class="number">15</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>第 1 点：我们知道，每个 bucket 有 8 个空位，在没有溢出，且所有的桶都装满了的情况下，装载因子算出来的结果是 8。因此当装载因子超过 6.5 时，表明很多 bucket 都快要装满了，查找效率和插入效率都变低了。在这个时候进行扩容是有必要的。</p><p>第 2 点：是对第 1 点的补充。就是说在装载因子比较小的情况下，这时候 map 的查找和插入效率也很低，而第 1 点识别不出来这种情况。表面现象就是计算装载因子的分子比较小，即 map 里元素总数少，但是 bucket 数量多（真实分配的 bucket 数量多，包括大量的 overflow bucket）。</p><p>不难想像造成这种情况的原因：不停地插入、删除元素。先插入很多元素，导致创建了很多 bucket，但是装载因子达不到第 1 点的临界值，未触发扩容来缓解这种情况。之后，删除元素降低元素总数量，再插入很多元素，导致创建很多的 overflow bucket，但就是不会触犯第 1 点的规定，你能拿我怎么办？overflow bucket 数量太多，导致 key 会很分散，查找插入效率低得吓人，因此出台第 2 点规定。这就像是一座空城，房子很多，但是住户很少，都分散了，找起人来很困难。</p><p><strong>对于命中条件 1，2 的限制，都会发生扩容。但是扩容的策略并不相同，毕竟两种条件应对的场景不同。</strong></p><p>对于条件 1，元素太多，而 bucket 数量太少，很简单：将 B 加 1，bucket 最大数量（2^B）直接变成原来 bucket 数量的 2 倍。于是，就有新老 bucket 了。注意，这时候元素都在老 bucket 里，还没迁移到新的 bucket 来。而且，新 bucket 只是最大数量变为原来最大数量（2^B）的 2 倍（2^B * 2）。</p><p>对于条件 2，其实元素没那么多，但是 overflow bucket 数特别多，说明很多 bucket 都没装满。解决办法就是开辟一个新 bucket 空间，将老 bucket 中的元素移动到新 bucket，使得同一个 bucket 中的 key 排列地更紧密。这样，原来，在 overflow bucket 中的 key 可以移动到 bucket 中来。结果是节省空间，提高 bucket 利用率，map 的查找和插入效率自然就会提升。</p><p>对于条件 2 的解决方案，还有一个极端的情况：如果插入 map 的 key 哈希都一样，就会落到同一个 bucket 里，超过 8 个就会产生 overflow bucket，结果也会造成 overflow bucket 数过多。移动元素其实解决不了问题，因为这时整个哈希表已经退化成了一个链表，操作效率变成了 <code>O(n)</code>。</p><p>再来看一下扩容具体是怎么做的。由于 map 扩容需要将原有的 key/value 重新搬迁到新的内存地址，如果有大量的 key/value 需要搬迁，会非常影响性能。因此 Go map 的扩容采取了一种称为“渐进式”的方式，原有的 key 并不会一次性搬迁完毕，每次最多只会搬迁 2 个 bucket。</p><p>上面说的 <code>hashGrow()</code> 函数实际上<strong>并没有真正地“搬迁”</strong>，它只是分配好了新的 buckets，并将老的 buckets 挂到了 oldbuckets 字段上。<strong>真正搬迁</strong> buckets 的动作在 <code>growWork()</code> 函数中，而调用 <code>growWork()</code> 函数的动作是在 mapassign 和 mapdelete 函数中。<strong>也就是插入或修改、删除 key 的时候，都会尝试进行搬迁 buckets 的工作</strong>。先检查 oldbuckets 是否搬迁完毕，具体来说就是检查 oldbuckets 是否为 nil。</p><p>我们先看 <code>hashGrow()</code> 函数所做的工作，再来看具体的搬迁 buckets 是如何进行的。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">hashGrow</span><span class="params">(t *maptype, h *hmap)</span></span> {</span><br><span class="line"> <span class="comment">// B+1 相当于是原来 2 倍的空间</span></span><br><span class="line"> bigger := <span class="keyword">uint8</span>(<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 对应条件 2</span></span><br><span class="line"> <span class="keyword">if</span> !overLoadFactor(<span class="keyword">int64</span>(h.count), h.B) {</span><br><span class="line"> <span class="comment">// 进行等量的内存扩容，所以 B 不变</span></span><br><span class="line"> bigger = <span class="number">0</span></span><br><span class="line"> h.flags |= sameSizeGrow</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 将老 buckets 挂到 buckets 上</span></span><br><span class="line"> oldbuckets := h.buckets</span><br><span class="line"> <span class="comment">// 申请新的 buckets 空间</span></span><br><span class="line"> newbuckets, nextOverflow := makeBucketArray(t, h.B+bigger)</span><br><span class="line"></span><br><span class="line"> flags := h.flags &^ (iterator | oldIterator)</span><br><span class="line"> <span class="keyword">if</span> h.flags&iterator != <span class="number">0</span> {</span><br><span class="line"> flags |= oldIterator</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 提交 grow 的动作</span></span><br><span class="line"> h.B += bigger</span><br><span class="line"> h.flags = flags</span><br><span class="line"> h.oldbuckets = oldbuckets</span><br><span class="line"> h.buckets = newbuckets</span><br><span class="line"> <span class="comment">// 搬迁进度为 0</span></span><br><span class="line"> h.nevacuate = <span class="number">0</span></span><br><span class="line"> <span class="comment">// overflow buckets 数为 0</span></span><br><span class="line"> h.noverflow = <span class="number">0</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// ……</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>运算符：&^。这叫<code>按位置 0</code>运算符</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">x = <span class="number">01010011</span></span><br><span class="line">y = <span class="number">01010100</span></span><br><span class="line">z = x &^ y = <span class="number">00000011</span></span><br></pre></td></tr></table></figure><p>如果 y bit 位为 1，那么结果 z 对应 bit 位就为 0，否则 z 对应 bit 位就和 x 对应 bit 位的值相同。</p><p>几个标志位如下：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 可能有迭代器使用 buckets</span></span><br><span class="line">iterator = <span class="number">1</span></span><br><span class="line"><span class="comment">// 可能有迭代器使用 oldbuckets</span></span><br><span class="line">oldIterator = <span class="number">2</span></span><br><span class="line"><span class="comment">// 有协程正在向 map 中写入 key</span></span><br><span class="line">hashWriting = <span class="number">4</span></span><br><span class="line"><span class="comment">// 等量扩容（对应条件 2）</span></span><br><span class="line">sameSizeGrow = <span class="number">8</span></span><br></pre></td></tr></table></figure><p>再来看看真正执行搬迁工作的 growWork() 函数</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">growWork</span><span class="params">(t *maptype, h *hmap, bucket <span class="keyword">uintptr</span>)</span></span> {</span><br><span class="line"> <span class="comment">// 确认搬迁老的 bucket 对应正在使用的 bucket</span></span><br><span class="line"> evacuate(t, h, bucket&h.oldbucketmask())</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 再搬迁一个 bucket，以加快搬迁进程</span></span><br><span class="line"> <span class="keyword">if</span> h.growing() {</span><br><span class="line"> evacuate(t, h, h.nevacuate)</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(h *hmap)</span> <span class="title">growing</span><span class="params">()</span> <span class="title">bool</span></span> {</span><br><span class="line"> <span class="keyword">return</span> h.oldbuckets != <span class="literal">nil</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p><code>bucket&h.oldbucketmask()</code> 这行代码，如源码注释里说的，是为了确认搬迁的 bucket 是我们正在使用的 bucket。</p><p>接下来，我们集中所有的精力在搬迁的关键函数 evacuate。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">evacuate</span><span class="params">(t *maptype, h *hmap, oldbucket <span class="keyword">uintptr</span>)</span></span> {</span><br><span class="line"> <span class="comment">// 定位老的 bucket 地址</span></span><br><span class="line"> b := (*bmap)(add(h.oldbuckets, oldbucket*<span class="keyword">uintptr</span>(t.bucketsize)))</span><br><span class="line"> <span class="comment">// 结果是 2^B，如 B = 5，结果为32</span></span><br><span class="line"> newbit := h.noldbuckets()</span><br><span class="line"> <span class="comment">// key 的哈希函数</span></span><br><span class="line"> alg := t.key.alg</span><br><span class="line"> <span class="comment">// 如果 b 没有被搬迁过</span></span><br><span class="line"> <span class="keyword">if</span> !evacuated(b) {</span><br><span class="line"> <span class="keyword">var</span> (</span><br><span class="line"> <span class="comment">// 表示bucket 移动的目标地址</span></span><br><span class="line"> x, y *bmap</span><br><span class="line"> <span class="comment">// 指向 x,y 中的 key/val</span></span><br><span class="line"> xi, yi <span class="keyword">int</span></span><br><span class="line"> <span class="comment">// 指向 x，y 中的 key</span></span><br><span class="line"> xk, yk unsafe.Pointer</span><br><span class="line"> <span class="comment">// 指向 x，y 中的 value</span></span><br><span class="line"> xv, yv unsafe.Pointer</span><br><span class="line"> )</span><br><span class="line"> <span class="comment">// 默认是等 size 扩容，前后 bucket 序号不变</span></span><br><span class="line"> <span class="comment">// 使用 x 来进行搬迁</span></span><br><span class="line"> x = (*bmap)(add(h.buckets, oldbucket*<span class="keyword">uintptr</span>(t.bucketsize)))</span><br><span class="line"> xi = <span class="number">0</span></span><br><span class="line"> xk = add(unsafe.Pointer(x), dataOffset)</span><br><span class="line"> xv = add(xk, bucketCnt*<span class="keyword">uintptr</span>(t.keysize))、</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果不是等 size 扩容，前后 bucket 序号有变</span></span><br><span class="line"> <span class="comment">// 使用 y 来进行搬迁</span></span><br><span class="line"> <span class="keyword">if</span> !h.sameSizeGrow() {</span><br><span class="line"> <span class="comment">// y 代表的 bucket 序号增加了 2^B</span></span><br><span class="line"> y = (*bmap)(add(h.buckets, (oldbucket+newbit)*<span class="keyword">uintptr</span>(t.bucketsize)))</span><br><span class="line"> yi = <span class="number">0</span></span><br><span class="line"> yk = add(unsafe.Pointer(y), dataOffset)</span><br><span class="line"> yv = add(yk, bucketCnt*<span class="keyword">uintptr</span>(t.keysize))</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 遍历所有的 bucket，包括 overflow buckets</span></span><br><span class="line"> <span class="comment">// b 是老的 bucket 地址</span></span><br><span class="line"> <span class="keyword">for</span> ; b != <span class="literal">nil</span>; b = b.overflow(t) {</span><br><span class="line"> k := add(unsafe.Pointer(b), dataOffset)</span><br><span class="line"> v := add(k, bucketCnt*<span class="keyword">uintptr</span>(t.keysize))</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 遍历 bucket 中的所有 cell</span></span><br><span class="line"> <span class="keyword">for</span> i := <span class="number">0</span>; i < bucketCnt; i, k, v = i+<span class="number">1</span>, add(k, <span class="keyword">uintptr</span>(t.keysize)), add(v, <span class="keyword">uintptr</span>(t.valuesize)) {</span><br><span class="line"> <span class="comment">// 当前 cell 的 top hash 值</span></span><br><span class="line"> top := b.tophash[i]</span><br><span class="line"> <span class="comment">// 如果 cell 为空，即没有 key</span></span><br><span class="line"> <span class="keyword">if</span> top == empty {</span><br><span class="line"> <span class="comment">// 那就标志它被"搬迁"过</span></span><br><span class="line"> b.tophash[i] = evacuatedEmpty</span><br><span class="line"> <span class="comment">// 继续下个 cell</span></span><br><span class="line"> <span class="keyword">continue</span></span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 正常不会出现这种情况</span></span><br><span class="line"> <span class="comment">// 未被搬迁的 cell 只可能是 empty 或是</span></span><br><span class="line"> <span class="comment">// 正常的 top hash（大于 minTopHash）</span></span><br><span class="line"> <span class="keyword">if</span> top < minTopHash {</span><br><span class="line"> throw(<span class="string">"bad map state"</span>)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> k2 := k</span><br><span class="line"> <span class="comment">// 如果 key 是指针，则解引用</span></span><br><span class="line"> <span class="keyword">if</span> t.indirectkey {</span><br><span class="line"> k2 = *((*unsafe.Pointer)(k2))</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 默认使用 X，等量扩容</span></span><br><span class="line"> useX := <span class="literal">true</span></span><br><span class="line"> <span class="comment">// 如果不是等量扩容</span></span><br><span class="line"> <span class="keyword">if</span> !h.sameSizeGrow() {</span><br><span class="line"> <span class="comment">// 计算 hash 值，和 key 第一次写入时一样</span></span><br><span class="line"> hash := alg.hash(k2, <span class="keyword">uintptr</span>(h.hash0))</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果有协程正在遍历 map</span></span><br><span class="line"> <span class="keyword">if</span> h.flags&iterator != <span class="number">0</span> {</span><br><span class="line"> <span class="comment">// 如果出现 相同的 key 值，算出来的 hash 值不同</span></span><br><span class="line"> <span class="keyword">if</span> !t.reflexivekey && !alg.equal(k2, k2) {</span><br><span class="line"> <span class="comment">// 只有在 float 变量的 NaN() 情况下会出现</span></span><br><span class="line"> <span class="keyword">if</span> top&<span class="number">1</span> != <span class="number">0</span> {</span><br><span class="line"> <span class="comment">// 第 B 位置 1</span></span><br><span class="line"> hash |= newbit</span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> <span class="comment">// 第 B 位置 0</span></span><br><span class="line"> hash &^= newbit</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 取高 8 位作为 top hash 值</span></span><br><span class="line"> top = <span class="keyword">uint8</span>(hash >> (sys.PtrSize*<span class="number">8</span> - <span class="number">8</span>))</span><br><span class="line"> <span class="keyword">if</span> top < minTopHash {</span><br><span class="line"> top += minTopHash</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 取决于新哈希值的 oldB+1 位是 0 还是 1</span></span><br><span class="line"> <span class="comment">// 详细看后面的文章</span></span><br><span class="line"> useX = hash&newbit == <span class="number">0</span></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 如果 key 搬到 X 部分</span></span><br><span class="line"> <span class="keyword">if</span> useX {</span><br><span class="line"> <span class="comment">// 标志老的 cell 的 top hash 值，表示搬移到 X 部分</span></span><br><span class="line"> b.tophash[i] = evacuatedX</span><br><span class="line"> <span class="comment">// 如果 xi 等于 8，说明要溢出了</span></span><br><span class="line"> <span class="keyword">if</span> xi == bucketCnt {</span><br><span class="line"> <span class="comment">// 新建一个 bucket</span></span><br><span class="line"> newx := h.newoverflow(t, x)</span><br><span class="line"> x = newx</span><br><span class="line"> <span class="comment">// xi 从 0 开始计数</span></span><br><span class="line"> xi = <span class="number">0</span></span><br><span class="line"> <span class="comment">// xk 表示 key 要移动到的位置</span></span><br><span class="line"> xk = add(unsafe.Pointer(x), dataOffset)</span><br><span class="line"> <span class="comment">// xv 表示 value 要移动到的位置</span></span><br><span class="line"> xv = add(xk, bucketCnt*<span class="keyword">uintptr</span>(t.keysize))</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 设置 top hash 值</span></span><br><span class="line"> x.tophash[xi] = top</span><br><span class="line"> <span class="comment">// key 是指针</span></span><br><span class="line"> <span class="keyword">if</span> t.indirectkey {</span><br><span class="line"> <span class="comment">// 将原 key（是指针）复制到新位置</span></span><br><span class="line"> *(*unsafe.Pointer)(xk) = k2 <span class="comment">// copy pointer</span></span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> <span class="comment">// 将原 key（是值）复制到新位置</span></span><br><span class="line"> typedmemmove(t.key, xk, k) <span class="comment">// copy value</span></span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// value 是指针，操作同 key</span></span><br><span class="line"> <span class="keyword">if</span> t.indirectvalue {</span><br><span class="line"> *(*unsafe.Pointer)(xv) = *(*unsafe.Pointer)(v)</span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> typedmemmove(t.elem, xv, v)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 定位到下一个 cell</span></span><br><span class="line"> xi++</span><br><span class="line"> xk = add(xk, <span class="keyword">uintptr</span>(t.keysize))</span><br><span class="line"> xv = add(xv, <span class="keyword">uintptr</span>(t.valuesize))</span><br><span class="line"> } <span class="keyword">else</span> { <span class="comment">// key 搬到 Y 部分，操作同 X 部分</span></span><br><span class="line"> <span class="comment">// ……</span></span><br><span class="line"> <span class="comment">// 省略了这部分，操作和 X 部分相同</span></span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 如果没有协程在使用老的 buckets，就把老 buckets 清除掉，帮助gc</span></span><br><span class="line"> <span class="keyword">if</span> h.flags&oldIterator == <span class="number">0</span> {</span><br><span class="line"> b = (*bmap)(add(h.oldbuckets, oldbucket*<span class="keyword">uintptr</span>(t.bucketsize)))</span><br><span class="line"> <span class="comment">// 只清除bucket 的 key,value 部分，保留 top hash 部分，指示搬迁状态</span></span><br><span class="line"> <span class="keyword">if</span> t.bucket.kind&kindNoPointers == <span class="number">0</span> {</span><br><span class="line"> memclrHasPointers(add(unsafe.Pointer(b), dataOffset), <span class="keyword">uintptr</span>(t.bucketsize)-dataOffset)</span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> memclrNoHeapPointers(add(unsafe.Pointer(b), dataOffset), <span class="keyword">uintptr</span>(t.bucketsize)-dataOffset)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 更新搬迁进度</span></span><br><span class="line"> <span class="comment">// 如果此次搬迁的 bucket 等于当前进度</span></span><br><span class="line"> <span class="keyword">if</span> oldbucket == h.nevacuate {</span><br><span class="line"> <span class="comment">// 进度加 1</span></span><br><span class="line"> h.nevacuate = oldbucket + <span class="number">1</span></span><br><span class="line"> <span class="comment">// Experiments suggest that 1024 is overkill by at least an order of magnitude.</span></span><br><span class="line"> <span class="comment">// Put it in there as a safeguard anyway, to ensure O(1) behavior.</span></span><br><span class="line"> <span class="comment">// 尝试往后看 1024 个 bucket</span></span><br><span class="line"> stop := h.nevacuate + <span class="number">1024</span></span><br><span class="line"> <span class="keyword">if</span> stop > newbit {</span><br><span class="line"> stop = newbit</span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 寻找没有搬迁的 bucket</span></span><br><span class="line"> <span class="keyword">for</span> h.nevacuate != stop && bucketEvacuated(t, h, h.nevacuate) {</span><br><span class="line"> h.nevacuate++</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 现在 h.nevacuate 之前的 bucket 都被搬迁完毕</span></span><br><span class="line"></span><br><span class="line"> <span class="comment">// 所有的 buckets 搬迁完毕</span></span><br><span class="line"> <span class="keyword">if</span> h.nevacuate == newbit {</span><br><span class="line"> <span class="comment">// 清除老的 buckets</span></span><br><span class="line"> h.oldbuckets = <span class="literal">nil</span></span><br><span class="line"> <span class="comment">// 清除老的 overflow bucket</span></span><br><span class="line"> <span class="comment">// 回忆一下：[0] 表示当前 overflow bucket</span></span><br><span class="line"> <span class="comment">// [1] 表示 old overflow bucket</span></span><br><span class="line"> <span class="keyword">if</span> h.extra != <span class="literal">nil</span> {</span><br><span class="line"> h.extra.overflow[<span class="number">1</span>] = <span class="literal">nil</span></span><br><span class="line"> }</span><br><span class="line"> <span class="comment">// 清除正在扩容的标志位</span></span><br><span class="line"> h.flags &^= sameSizeGrow</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>evacuate 函数的代码注释非常清晰，对着代码和注释是很容易看懂整个的搬迁过程的，耐心点。</p><p>搬迁的目的就是将老的 buckets 搬迁到新的 buckets。而通过前面的说明我们知道，应对条件 1，新的 buckets 数量是之前的一倍，应对条件 2，新的 buckets 数量和之前相等。</p><p>对于条件 2，从老的 buckets 搬迁到新的 buckets，由于 bucktes 数量不变，因此可以按序号来搬，比如原来在 0 号 bucktes，到新的地方后，仍然放在 0 号 buckets。</p><p>对于条件 1，就没这么简单了。要重新计算 key 的哈希，才能决定它到底落在哪个 bucket。例如，原来 B = 5，计算出 key 的哈希后，只用看它的低 5 位，就能决定它落在哪个 bucket。扩容后，B 变成了 6，因此需要多看一位，它的低 6 位决定 key 落在哪个 bucket。这称为 <code>rehash</code>。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20220104232034372.png" alt="image-20220104232034372"></p><p>因此，某个 key 在搬迁前后 bucket 序号可能和原来相等，也可能是相比原来加上 2^B（原来的 B 值），取决于 hash 值 第 6 bit 位是 0 还是 1。</p><p>再明确一个问题：如果扩容后，B 增加了 1，意味着 buckets 总数是原来的 2 倍，原来 1 号的桶“裂变”到两个桶。</p><p>例如，原始 B = 2，1 号 bucket 中有 2 个 key 的哈希值低 3 位分别为：010，110。由于原来 B = 2，所以低 2 位 <code>10</code> 决定它们落在 2 号桶，现在 B 变成 3，所以 <code>010</code>、<code>110</code> 分别落入 2、6 号桶。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104232110233.png" alt="image-20220104232110233"></p><p>关键点:</p><p>evacuate 函数每次只完成一个 bucket 的搬迁工作，因此要遍历完此 bucket 的所有的 cell，将有值的 cell copy 到新的地方。bucket 还会链接 overflow bucket，它们同样需要搬迁。因此会有 2 层循环，外层遍历 bucket 和 overflow bucket，内层遍历 bucket 的所有 cell。这样的循环在 map 的源码里到处都是，要理解透了。</p><p>源码里提到 X, Y part，其实就是我们说的如果是扩容到原来的 2 倍，桶的数量是原来的 2 倍，前一半桶被称为 X part，后一半桶被称为 Y part。一个 bucket 中的 key 可能会分裂落到 2 个桶，一个位于 X part，一个位于 Y part。所以在搬迁一个 cell 之前，需要知道这个 cell 中的 key 是落到哪个 Part。很简单，重新计算 cell 中 key 的 hash，并向前“多看”一位，决定落入哪个 Part，这个前面也说得很详细了。</p><p>有一个特殊情况是：有一种 key，每次对它计算 hash，得到的结果都不一样。这个 key 就是 <code>math.NaN()</code> 的结果，它的含义是 <code>not a number</code>，类型是 float64。当它作为 map 的 key，在搬迁的时候，会遇到一个问题：再次计算它的哈希值和它当初插入 map 时的计算出来的哈希值不一样！</p><p>你可能想到了，这样带来的一个后果是，这个 key 是永远不会被 Get 操作获取的！当我使用 <code>m[math.NaN()]</code> 语句的时候，是查不出来结果的。这个 key 只有在遍历整个 map 的时候，才有机会现身。所以，可以向一个 map 插入任意数量的 <code>math.NaN()</code> 作为 key。</p><p>当搬迁碰到 <code>math.NaN()</code> 的 key 时，只通过 tophash 的最低位决定分配到 X part 还是 Y part（如果扩容后是原来 buckets 数量的 2 倍）。如果 tophash 的最低位是 0 ，分配到 X part；如果是 1 ，则分配到 Y part。</p><p>确定了要搬迁到的目标 bucket 后，搬迁操作就比较好进行了。将源 key/value 值 copy 到目的地相应的位置。</p><p>设置 key 在原始 buckets 的 tophash 为 <code>evacuatedX</code> 或是 <code>evacuatedY</code>，表示已经搬迁到了新 map 的 x part 或是 y part。新 map 的 tophash 则正常取 key 哈希值的高 8 位。</p><p>扩容前，B = 2，共有 4 个 buckets，lowbits 表示 hash 值的低位(B 为 2，都是 10,在同一个 bucket 中)。假设我们不关注其他 buckets 情况，专注在 2 号 bucket。并且假设 overflow 太多，触发了等量扩容（对应于前面的条件 2）。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104232812869.png" alt="image-20220104232812869"></p><p>扩容完成后，overflow bucket 消失了，key 都集中到了一个 bucket，更为紧凑了，提高了查找的效率。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104232911854.png" alt="image-20220104232911854"></p><p>假设触发了 2 倍的扩容，那么扩容完成后，老 buckets 中的 key 分裂到了 2 个 新的 bucket。一个在 x part，一个在 y 的 part。依据是 hash 的 lowbits。新 map 中 <code>0-3</code> 称为 x part，<code>4-7</code> 称为 y part。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20220104232940068.png" alt="image-20220104232940068"></p><p>注意，上面的两张图忽略了其他 buckets 的搬迁情况，表示所有的 bucket 都搬迁完毕后的情形。实际上，我们知道，搬迁是一个“渐进”的过程，并不会一下子就全部搬迁完毕。所以在搬迁过程中，oldbuckets 指针还会指向原来老的 []bmap，并且已经搬迁完毕的 key 的 tophash 值会是一个状态值，表示 key 的搬迁去向。</p>]]></content>
<summary type="html"><h2 id="Map"><a href="#Map" class="headerlink" title="Map"></a>Map</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// A header for a Go map.</span></span><br><span class="line"><span class="keyword">type</span> hmap <span class="keyword">struct</span> &#123;</span><br><span class="line"> <span class="comment">// 元素个数，调用 len(map) 时，直接返回此值</span></span><br><span class="line"> count <span class="keyword">int</span></span><br><span class="line"> flags <span class="keyword">uint8</span></span><br><span class="line"> <span class="comment">// buckets 的对数</span></span><br><span class="line"> B <span class="keyword">uint8</span></span><br><span class="line"> <span class="comment">// overflow 的 bucket 近似数</span></span><br><span class="line"> noverflow <span class="keyword">uint16</span></span><br><span class="line"> <span class="comment">// 计算 key 的哈希的时候会传入哈希函数</span></span><br><span class="line"> hash0 <span class="keyword">uint32</span></span><br><span class="line"> <span class="comment">// 指向 buckets 数组，大小为 2^B</span></span><br><span class="line"> <span class="comment">// 如果元素个数为0，就为 nil</span></span><br><span class="line"> buckets unsafe.Pointer</span><br><span class="line"> <span class="comment">// 扩容的时候，buckets 长度会是 oldbuckets 的两倍</span></span><br><span class="line"> oldbuckets unsafe.Pointer</span><br><span class="line"> <span class="comment">// 指示扩容进度，小于此地址的 buckets 迁移完成</span></span><br><span class="line"> nevacuate <span class="keyword">uintptr</span></span><br><span class="line"> extra *mapextra <span class="comment">// optional fields</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></summary>
<category term="Go" scheme="https://cwww3.github.io/tags/Go/"/>
</entry>
<entry>
<title>slice</title>
<link href="https://cwww3.github.io/2021/12/31/slice/"/>
<id>https://cwww3.github.io/2021/12/31/slice/</id>
<published>2021-12-31T09:40:02.000Z</published>
<updated>2021-12-31T10:00:13.163Z</updated>
<content type="html"><![CDATA[<h2 id="Slice"><a href="#Slice" class="headerlink" title="Slice"></a>Slice</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> SliceHeader <span class="keyword">struct</span> {</span><br><span class="line">Data <span class="keyword">uintptr</span></span><br><span class="line">Len <span class="keyword">int</span></span><br><span class="line">Cap <span class="keyword">int</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><span id="more"></span><h3 id="初始化"><a href="#初始化" class="headerlink" title="初始化"></a>初始化</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">arr[<span class="number">0</span>:<span class="number">3</span>] or slice[<span class="number">0</span>:<span class="number">3</span>]</span><br><span class="line">slice := []<span class="keyword">int</span>{<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>}</span><br><span class="line">slice := <span class="built_in">make</span>([]<span class="keyword">int</span>, <span class="number">10</span>)</span><br></pre></td></tr></table></figure><ul><li>Empty</li></ul><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 底层数据长度为0</span></span><br><span class="line">nums := <span class="built_in">make</span>([]<span class="keyword">int</span>,<span class="number">0</span>)</span><br><span class="line">nums := []<span class="keyword">int</span>{}</span><br></pre></td></tr></table></figure><ul><li>Nil</li></ul><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 未分配底层数组</span></span><br><span class="line"><span class="keyword">var</span> nums []<span class="keyword">int</span></span><br></pre></td></tr></table></figure><ul><li>empty和nil都可以使用append，底层会申请内存进行扩容</li><li>字面量创建</li></ul><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">var</span> vstat [<span class="number">3</span>]<span class="keyword">int</span> <span class="comment">// 1.根据切片中的元素数量对底层数组的大小进行推断并创建一个数组；</span></span><br><span class="line">vstat[<span class="number">0</span>] = <span class="number">1</span> <span class="comment">// 2.将这些字面量元素存储到初始化的数组中；</span></span><br><span class="line">vstat[<span class="number">1</span>] = <span class="number">2</span></span><br><span class="line">vstat[<span class="number">2</span>] = <span class="number">3</span></span><br><span class="line"><span class="keyword">var</span> vauto *[<span class="number">3</span>]<span class="keyword">int</span> = <span class="built_in">new</span>([<span class="number">3</span>]<span class="keyword">int</span>) <span class="comment">// 3.创建一个同样指向 [3]int 类型的数组指针；</span></span><br><span class="line">*vauto = vstat <span class="comment">// 4.将静态存储区的数组 vstat 赋值给 vauto 指针所在的地址；</span></span><br><span class="line">slice := vauto[:] <span class="comment">//5.通过 [:] 操作获取一个底层使用 vauto 的切片；</span></span><br></pre></td></tr></table></figure><p> [:] 就是使用下标创建切片的方法，从这一点也能看出 [:] 操作是创建切片最底层的一种方法。</p><ul><li>关键字创建</li></ul><p>如果使用字面量的方式创建切片，大部分的工作都会在编译期间完成。但是当我们使用 <code>make</code> 关键字创建切片时，很多工作都需要运行时的参与；调用方必须向 <code>make</code> 函数传入切片的大小以及可选的容量。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">1.</span> 检查<span class="built_in">len</span>是否传入，检查<span class="built_in">len</span>是否小于<span class="built_in">cap</span></span><br><span class="line"><span class="number">2.</span> 当切片发生逃逸或者非常大时，运行时需要 runtime.makeslice 在堆上初始化切片</span><br><span class="line"><span class="number">3.</span> 当前的切片不会发生逃逸并且切片非常小时，会使用与字面量相同的方式创建</span><br></pre></td></tr></table></figure><p><strong>使用 makeslice 创建切片</strong></p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">makeslice</span><span class="params">(et *_type, <span class="built_in">len</span>, <span class="built_in">cap</span> <span class="keyword">int</span>)</span> <span class="title">unsafe</span>.<span class="title">Pointer</span></span> {</span><br><span class="line">mem, overflow := math.MulUintptr(et.size, <span class="keyword">uintptr</span>(<span class="built_in">cap</span>))</span><br><span class="line"><span class="keyword">if</span> overflow || mem > maxAlloc || <span class="built_in">len</span> < <span class="number">0</span> || <span class="built_in">len</span> > <span class="built_in">cap</span> {</span><br><span class="line">mem, overflow := math.MulUintptr(et.size, <span class="keyword">uintptr</span>(<span class="built_in">len</span>))</span><br><span class="line"><span class="keyword">if</span> overflow || mem > maxAlloc || <span class="built_in">len</span> < <span class="number">0</span> {</span><br><span class="line">panicmakeslicelen()</span><br><span class="line">}</span><br><span class="line">panicmakeslicecap()</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="keyword">return</span> mallocgc(mem, et, <span class="literal">true</span>)</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>计算切片所需空间并在堆上申请一段连续的内存 (内存空间=切片中元素大小×切片容量)</p><p>如果发生一下情况，会崩溃</p><ul><li>内存空间的大小发生了溢出；</li><li>申请的内存大于最大可分配的内存；</li><li>传入的长度小于 0 或者长度大于容量；</li></ul><p><code>mallocgc</code> 是用于申请内存的函数，遇到了比较小的对象会直接初始化在 Go 语言调度器里面的 P 结构中，而大于 32KB 的对象会在堆上初始化。</p><h3 id="追加和扩容"><a href="#追加和扩容" class="headerlink" title="追加和扩容"></a>追加和扩容</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">a := <span class="built_in">make</span>([]<span class="keyword">int</span>,<span class="number">0</span>,<span class="number">1</span>)</span><br><span class="line"><span class="comment">// 当容量足够时，直接添加</span></span><br><span class="line">a = <span class="built_in">append</span>(a,<span class="number">1</span>)</span><br><span class="line"><span class="comment">// 当容量不足时，先扩容(拷贝原先的数据),再添加 这是底层数组已经改变</span></span><br><span class="line">a = <span class="built_in">append</span>(a,<span class="number">2</span>)</span><br></pre></td></tr></table></figure><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">growslice</span><span class="params">(et *_type, old slice, <span class="built_in">cap</span> <span class="keyword">int</span>)</span> <span class="title">slice</span></span> {</span><br><span class="line"> ...</span><br><span class="line"> newcap := old.<span class="built_in">cap</span></span><br><span class="line"> doublecap := newcap + newcap</span><br><span class="line"> <span class="keyword">if</span> <span class="built_in">cap</span> > doublecap {</span><br><span class="line"> newcap = <span class="built_in">cap</span></span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> <span class="keyword">if</span> old.<span class="built_in">len</span> < <span class="number">1024</span> {</span><br><span class="line"> newcap = doublecap</span><br><span class="line"> } <span class="keyword">else</span> {</span><br><span class="line"> <span class="keyword">for</span> <span class="number">0</span> < newcap && newcap < <span class="built_in">cap</span> {</span><br><span class="line"> newcap += newcap / <span class="number">4</span></span><br><span class="line"> }</span><br><span class="line"> ...</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> ...</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>在分配内存空间之前需要先确定新的切片容量，运行时根据切片的当前容量选择不同的策略进行扩容：</p><ul><li>如果期望容量大于当前容量的两倍就会使用期望容量；</li><li>如果当前切片的长度小于 1024 就会将容量翻倍；</li><li>如果当前切片的长度大于 1024 就会每次增加 25% 的容量，直到新容量大于期望容量；</li></ul><p>上述代码片段仅会确定切片的<strong>大致容量</strong>，下面还需要根据切片中的元素大小<strong>对齐内存</strong>，当数组中元素所占的字节大小为 1、8 或者 2 的倍数时，运行时会使用如下所示的代码对齐内存：</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">var</span> overflow <span class="keyword">bool</span></span><br><span class="line"><span class="keyword">var</span> lenmem, newlenmem, capmem <span class="keyword">uintptr</span></span><br><span class="line"><span class="keyword">switch</span> {</span><br><span class="line"><span class="keyword">case</span> et.size == <span class="number">1</span>:</span><br><span class="line">lenmem = <span class="keyword">uintptr</span>(old.<span class="built_in">len</span>)</span><br><span class="line">newlenmem = <span class="keyword">uintptr</span>(<span class="built_in">cap</span>)</span><br><span class="line">capmem = roundupsize(<span class="keyword">uintptr</span>(newcap))</span><br><span class="line">overflow = <span class="keyword">uintptr</span>(newcap) > maxAlloc</span><br><span class="line">newcap = <span class="keyword">int</span>(capmem)</span><br><span class="line"><span class="keyword">case</span> et.size == sys.PtrSize:</span><br><span class="line">lenmem = <span class="keyword">uintptr</span>(old.<span class="built_in">len</span>) * sys.PtrSize</span><br><span class="line">newlenmem = <span class="keyword">uintptr</span>(<span class="built_in">cap</span>) * sys.PtrSize</span><br><span class="line">capmem = roundupsize(<span class="keyword">uintptr</span>(newcap) * sys.PtrSize)</span><br><span class="line">overflow = <span class="keyword">uintptr</span>(newcap) > maxAlloc/sys.PtrSize</span><br><span class="line">newcap = <span class="keyword">int</span>(capmem / sys.PtrSize)</span><br><span class="line"><span class="keyword">case</span> isPowerOfTwo(et.size):</span><br><span class="line">...</span><br><span class="line"><span class="keyword">default</span>:</span><br><span class="line">...</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="comment">// roundupsize函数会将待申请的内存向上取整 取整时会使用 runtime.class_to_size 数组</span></span><br><span class="line"><span class="keyword">var</span> class_to_size = [_NumSizeClasses]<span class="keyword">uint16</span>{<span class="number">0</span>,<span class="number">8</span>,<span class="number">16</span>,<span class="number">32</span>,<span class="number">48</span>,<span class="number">64</span>,<span class="number">80</span>,...}</span><br></pre></td></tr></table></figure><p><strong>例子</strong></p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">a = []<span class="keyword">int</span>{}</span><br><span class="line">a = a.<span class="built_in">append</span>(<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,<span class="number">4</span>,<span class="number">5</span>) <span class="comment">// len=5 cap=6</span></span><br><span class="line"><span class="comment">// 期望容量(5)大于原容量的两倍(2*0) 得到新容量(5) 占用内存(5*8byte) </span></span><br><span class="line"><span class="comment">// 进行内存对齐得到（48）最终容量(48/8==6)</span></span><br><span class="line"></span><br><span class="line">a = []<span class="keyword">int</span>{<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,<span class="number">4</span>}</span><br><span class="line">a = a.<span class="built_in">append</span>(<span class="number">5</span>) <span class="comment">// len=5 cap=8</span></span><br><span class="line"><span class="comment">// 期望容量(5)小于1024 (2*0) 新容量为原容量的两倍(8) 占用内存(8*8byte) </span></span><br><span class="line"><span class="comment">// 进行内存对齐得到（64）最终容量(64/8==8)</span></span><br></pre></td></tr></table></figure><h3 id="拷贝切片"><a href="#拷贝切片" class="headerlink" title="拷贝切片"></a>拷贝切片</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 将src的内容拷贝到dst中 拷贝的大小取两者中的较小值</span></span><br><span class="line"><span class="built_in">copy</span>(dst, src)</span><br></pre></td></tr></table></figure><h3 id="传值和传指针"><a href="#传值和传指针" class="headerlink" title="传值和传指针"></a>传值和传指针</h3><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> main</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> <span class="string">"fmt"</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">myAppend</span><span class="params">(s []<span class="keyword">int</span>)</span> []<span class="title">int</span></span> {</span><br><span class="line"><span class="comment">// 这里 s 虽然改变了，但并不会影响外层函数的 s</span></span><br><span class="line">s = <span class="built_in">append</span>(s, <span class="number">100</span>)</span><br><span class="line"><span class="keyword">return</span> s</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">myAppendPtr</span><span class="params">(s *[]<span class="keyword">int</span>)</span></span> {</span><br><span class="line"><span class="comment">// 会改变外层 s 本身</span></span><br><span class="line">*s = <span class="built_in">append</span>(*s, <span class="number">100</span>)</span><br><span class="line"><span class="keyword">return</span></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> {</span><br><span class="line">s := []<span class="keyword">int</span>{<span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>}</span><br><span class="line">newS := myAppend(s) </span><br><span class="line"></span><br><span class="line">fmt.Println(s) <span class="comment">// 1,1,1</span></span><br><span class="line">fmt.Println(newS) <span class="comment">// 1,1,1,100</span></span><br><span class="line"></span><br><span class="line">s = newS</span><br><span class="line"></span><br><span class="line">myAppendPtr(&s)</span><br><span class="line">fmt.Println(s) <span class="comment">// 1,1,1,100,1000</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><h2 id="Slice"><a href="#Slice" class="headerlink" title="Slice"></a>Slice</h2><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> SliceHeader <span class="keyword">struct</span> &#123;</span><br><span class="line"> Data <span class="keyword">uintptr</span></span><br><span class="line"> Len <span class="keyword">int</span></span><br><span class="line"> Cap <span class="keyword">int</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure></summary>
<category term="Go" scheme="https://cwww3.github.io/tags/Go/"/>
</entry>
<entry>
<title>kqueue</title>
<link href="https://cwww3.github.io/2021/12/24/kqueue/"/>
<id>https://cwww3.github.io/2021/12/24/kqueue/</id>
<published>2021-12-23T17:06:14.000Z</published>
<updated>2022-09-03T18:36:37.818Z</updated>
<content type="html"><![CDATA[<h2 id="使用-Kqueue-实现简单的-TCP-服务器"><a href="#使用-Kqueue-实现简单的-TCP-服务器" class="headerlink" title="使用 Kqueue 实现简单的 TCP 服务器"></a>使用 Kqueue 实现简单的 TCP 服务器</h2><p><a href="https://github.com/FRosner/FrSrv">项目地址</a></p><h3 id="设计"><a href="#设计" class="headerlink" title="设计"></a>设计</h3><p>TCP 服务器由 TCP socket，来自客户端连接的 socket，kqueue 以及轮询 kqueue 的 event loop 组成。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211223235715574.png" alt="image-20211223235715574"></p><span id="more"></span><p>当一个客户端想要连接服务器，连接请求会被放入 TCP 连接队列。内核会将该事件放入 kqueue，然后 event loop 轮询 kqueue 获取该事件，并创建一个新的客户端 socket。</p><p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211224000209004.png" alt="image-20211224000209004"></p><p>当客户端写入数据，内核会将该事件放入 kqueue，然后 event loop 轮询获取该事件，获取到 socket，并从这个 socket 中读取数据。</p><h3 id="实现"><a href="#实现" class="headerlink" title="实现"></a>实现</h3><ol><li>Create, bind, and listen on a new socket</li><li>Create new kqueue</li><li>Subscribe to socket events</li><li>Poll for new events in a loop and handle them</li></ol><p>socket 模块用于封装 socket 相关的所用功能，kqueue 模块用于封装 event loop 的所用功能。</p><p>在 main 模块中调用他们，实现 tcp 服务。</p><h4 id="socket"><a href="#socket" class="headerlink" title="socket"></a>socket</h4><p>用 Go 类型定义一个 Socket，需要将文件描述符进行保存。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> Socket <span class="keyword">struct</span> {</span><br><span class="line"> FileDescriptor <span class="keyword">int</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>实现读 io.Reader，写 io.Writer，关闭 io.Closer 等接口。</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(socket Socket)</span> <span class="title">Read</span><span class="params">(bytes []<span class="keyword">byte</span>)</span> <span class="params">(<span class="keyword">int</span>, error)</span></span> {</span><br><span class="line"> <span class="keyword">if</span> <span class="built_in">len</span>(bytes) == <span class="number">0</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="number">0</span>, <span class="literal">nil</span></span><br><span class="line"> }</span><br><span class="line"> numBytesRead, err :=</span><br><span class="line"> syscall.Read(socket.FileDescriptor, bytes)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> numBytesRead = <span class="number">0</span></span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">return</span> numBytesRead, err</span><br><span class="line">}</span><br></pre></td></tr></table></figure><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(socket Socket)</span> <span class="title">Write</span><span class="params">(bytes []<span class="keyword">byte</span>)</span> <span class="params">(<span class="keyword">int</span>, error)</span></span> {</span><br><span class="line"> numBytesWritten, err :=</span><br><span class="line"> syscall.Write(socket.FileDescriptor, bytes)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> numBytesWritten = <span class="number">0</span></span><br><span class="line"> }</span><br><span class="line"> <span class="keyword">return</span> numBytesWritten, err</span><br><span class="line">}</span><br></pre></td></tr></table></figure><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(socket *Socket)</span> <span class="title">Close</span><span class="params">()</span> <span class="title">error</span></span> {</span><br><span class="line"> <span class="keyword">return</span> syscall.Close(socket.FileDescriptor)</span><br><span class="line">}</span><br></pre></td></tr></table></figure><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(socket *Socket)</span> <span class="title">String</span><span class="params">()</span> <span class="title">string</span></span> {</span><br><span class="line"> <span class="keyword">return</span> strconv.Itoa(socket.FileDescriptor)</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>根据提供的 ip 和端口，创建并监听</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">Listen</span><span class="params">(ip <span class="keyword">string</span>, port <span class="keyword">int</span>)</span> <span class="params">(*Socket, error)</span></span> {</span><br><span class="line"> <span class="comment">// 1. 创建套接字</span></span><br><span class="line"> socket := &Socket{}</span><br><span class="line"><span class="comment">// AF_INET 表示IPV4</span></span><br><span class="line"> <span class="comment">// SOCK_STREAM 表示有序的、可靠的、基于双向连接的字节流</span></span><br><span class="line"> <span class="comment">// 0在SOCK_STREAM套接字中表示TCP</span></span><br><span class="line"> socketFileDescriptor, err :=</span><br><span class="line"> syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, <span class="number">0</span>)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>, fmt.Errorf(<span class="string">"failed to create socket (%v)"</span>, err)</span><br><span class="line"> }</span><br><span class="line"> socket.FileDescriptor = socketFileDescriptor</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 2. 绑定ip和端口</span></span><br><span class="line"> socketAddress := &syscall.SockaddrInet4{Port: port}</span><br><span class="line"> <span class="built_in">copy</span>(socketAddress.Addr[:], net.ParseIP(ip))</span><br><span class="line"> <span class="keyword">if</span> err = syscall.Bind(socket.FileDescriptor, socketAddress);</span><br><span class="line"> err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>, fmt.Errorf(<span class="string">"failed to bind socket (%v)"</span>, err)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 3. 开始监听,接收连接请求 SOMAXCONN设置最大等待的连接数</span></span><br><span class="line"> <span class="keyword">if</span> err = syscall.Listen(socket.FileDescriptor, syscall.SOMAXCONN);</span><br><span class="line"> err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>, fmt.Errorf(<span class="string">"failed to listen on socket (%v)"</span>, err)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> socket, <span class="literal">nil</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="kqueue"><a href="#kqueue" class="headerlink" title="kqueue"></a>kqueue</h4><p>定义一个 EventLoop 结构体</p><p>分别保存 kqueue 文件描述符以及用于监听客户端连接的套接字文件描述符</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">type</span> EventLoop <span class="keyword">struct</span> {</span><br><span class="line"> KqueueFileDescriptor <span class="keyword">int</span></span><br><span class="line"> SocketFileDescriptor <span class="keyword">int</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>定义一个创建 EventLoop 的函数</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">NewEventLoop</span><span class="params">(s *socket.Socket)</span> <span class="params">(*EventLoop, error)</span></span> {</span><br><span class="line"> <span class="comment">// 创建kqueue,返回对应的文件描述符</span></span><br><span class="line"> kQueue, err := syscall.Kqueue()</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>,</span><br><span class="line"> fmt.Errorf(<span class="string">"failed to create kqueue file descriptor (%v)"</span>, err)</span><br><span class="line"> }</span><br><span class="line"><span class="comment">// 定义监听的事件类型以及处理方法</span></span><br><span class="line"> <span class="comment">// Ident设置创建的socket文件描述符</span></span><br><span class="line"> <span class="comment">// Filter 设置成EVFILT_READ表示对传入的连接事件感兴趣</span></span><br><span class="line"> <span class="comment">// Flags 设置相应的响应动作 EV_ADD表示添加到kqueue中，EV_ENABLE表示启用</span></span><br><span class="line"> changeEvent := syscall.Kevent_t{</span><br><span class="line"> Ident: <span class="keyword">uint64</span>(s.FileDescriptor),</span><br><span class="line"> Filter: syscall.EVFILT_READ,</span><br><span class="line"> Flags: syscall.EV_ADD | syscall.EV_ENABLE,</span><br><span class="line"> Fflags: <span class="number">0</span>,</span><br><span class="line"> Data: <span class="number">0</span>,</span><br><span class="line"> Udata: <span class="literal">nil</span>,</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 将定义的事件类型注册要kqueue上</span></span><br><span class="line"> changeEventRegistered, err := syscall.Kevent(</span><br><span class="line"> kQueue,</span><br><span class="line"> []syscall.Kevent_t{changeEvent},</span><br><span class="line"> <span class="literal">nil</span>,</span><br><span class="line"> <span class="literal">nil</span></span><br><span class="line"> )</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> || changeEventRegistered == <span class="number">-1</span> {</span><br><span class="line"> <span class="keyword">return</span> <span class="literal">nil</span>,</span><br><span class="line"> fmt.Errorf(<span class="string">"failed to register change event (%v)"</span>, err)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="keyword">return</span> &EventLoop{</span><br><span class="line"> KqueueFileDescriptor: kQueue,</span><br><span class="line"> SocketFileDescriptor: s.FileDescriptor</span><br><span class="line"> }, <span class="literal">nil</span></span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>定义函数对 kquue 轮询处理</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="params">(eventLoop *EventLoop)</span> <span class="title">Handle</span><span class="params">(handler Handler)</span></span> {</span><br><span class="line"> <span class="keyword">for</span> {</span><br><span class="line"> <span class="comment">// 轮询kqueue获取事件的数量</span></span><br><span class="line"> newEvents := <span class="built_in">make</span>([]syscall.Kevent_t, <span class="number">10</span>)</span><br><span class="line"> numNewEvents, err := syscall.Kevent(</span><br><span class="line"> eventLoop.KqueueFileDescriptor,</span><br><span class="line"> <span class="literal">nil</span>,</span><br><span class="line"> newEvents,</span><br><span class="line"> <span class="literal">nil</span></span><br><span class="line"> )</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">continue</span></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 处理每一个事件</span></span><br><span class="line"> <span class="keyword">for</span> i := <span class="number">0</span>; i < numNewEvents; i++ {</span><br><span class="line"> currentEvent := newEvents[i]</span><br><span class="line"> eventFileDescriptor := <span class="keyword">int</span>(currentEvent.Ident)</span><br><span class="line"></span><br><span class="line"> <span class="keyword">if</span> currentEvent.Flags&syscall.EV_EOF != <span class="number">0</span> {</span><br><span class="line"> <span class="comment">// 客户端关闭事件</span></span><br><span class="line"> syscall.Close(eventFileDescriptor)</span><br><span class="line"> } <span class="keyword">else</span> <span class="keyword">if</span> eventFileDescriptor == eventLoop.SocketFileDescriptor {</span><br><span class="line"> <span class="comment">// 客户端连接事件</span></span><br><span class="line"> socketConnection, _, err :=</span><br><span class="line"> syscall.Accept(eventFileDescriptor)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> <span class="keyword">continue</span></span><br><span class="line"> }</span><br><span class="line"><span class="comment">// 获取客户端连接请求创建对应的套接字和文件描述符</span></span><br><span class="line"> <span class="comment">// 并定义订阅读事件，注册到kqueue上</span></span><br><span class="line"> socketEvent := syscall.Kevent_t{</span><br><span class="line"> Ident: <span class="keyword">uint64</span>(socketConnection),</span><br><span class="line"> Filter: syscall.EVFILT_READ,</span><br><span class="line"> Flags: syscall.EV_ADD,</span><br><span class="line"> Fflags: <span class="number">0</span>,</span><br><span class="line"> Data: <span class="number">0</span>,</span><br><span class="line"> Udata: <span class="literal">nil</span>,</span><br><span class="line"> }</span><br><span class="line"> socketEventRegistered, err := syscall.Kevent(</span><br><span class="line"> eventLoop.KqueueFileDescriptor,</span><br><span class="line"> []syscall.Kevent_t{socketEvent},</span><br><span class="line"> <span class="literal">nil</span>,</span><br><span class="line"> <span class="literal">nil</span></span><br><span class="line"> )</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> || socketEventRegistered == <span class="number">-1</span> {</span><br><span class="line"> <span class="keyword">continue</span></span><br><span class="line"> }</span><br><span class="line"> } <span class="keyword">else</span> <span class="keyword">if</span> currentEvent.Filter&syscall.EVFILT_READ != <span class="number">0</span> {</span><br><span class="line"> <span class="comment">// 处理已经建立连接的客户端所发来的数据</span></span><br><span class="line"> handler(&socket.Socket{</span><br><span class="line"> FileDescriptor: <span class="keyword">int</span>(eventFileDescriptor)</span><br><span class="line"> })</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> <span class="comment">// ignore all other events</span></span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure><h4 id="main"><a href="#main" class="headerlink" title="main"></a>main</h4><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> {</span><br><span class="line"> s, err := socket.Listen(<span class="string">"127.0.0.1"</span>, <span class="number">8080</span>)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> log.Println(<span class="string">"Failed to create Socket:"</span>, err)</span><br><span class="line"> os.Exit(<span class="number">1</span>)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> eventLoop, err := kqueue.NewEventLoop(s)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> {</span><br><span class="line"> log.Println(<span class="string">"Failed to create event loop:"</span>, err)</span><br><span class="line"> os.Exit(<span class="number">1</span>)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> log.Println(<span class="string">"Server started. Waiting for incoming connections. ^C to exit."</span>)</span><br><span class="line"></span><br><span class="line"> <span class="comment">// 定义以建立连接的客户端发来数据时的处理函数</span></span><br><span class="line"> eventLoop.Handle(<span class="function"><span class="keyword">func</span><span class="params">(s *socket.Socket)</span></span> {</span><br><span class="line"> reader := bufio.NewReader(s)</span><br><span class="line"> <span class="keyword">for</span> {</span><br><span class="line"> line, err := reader.ReadString(<span class="string">'\n'</span>)</span><br><span class="line"> <span class="keyword">if</span> err != <span class="literal">nil</span> || strings.TrimSpace(line) == <span class="string">""</span> {</span><br><span class="line"> <span class="keyword">break</span></span><br><span class="line"> }</span><br><span class="line"> s.Write([]<span class="keyword">byte</span>(line))</span><br><span class="line"> }</span><br><span class="line"> s.Close()</span><br><span class="line"> })</span><br><span class="line">}</span><br></pre></td></tr></table></figure><p>模拟连接、发送数据、关闭连接</p><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">func</span> <span class="title">main</span><span class="params">()</span></span> {</span><br><span class="line">c,err := net.Dial(<span class="string">"tcp"</span>,<span class="string">"127.0.0.1:8080"</span>)</span><br><span class="line"><span class="keyword">if</span> err!= <span class="literal">nil</span> {</span><br><span class="line">log.Fatalln(err)</span><br><span class="line">}</span><br><span class="line"><span class="keyword">defer</span> c.Close()</span><br><span class="line">_,_ = c.Write([]<span class="keyword">byte</span>(<span class="string">"hello world\n"</span>))</span><br><span class="line">log.Println(<span class="string">"send msg"</span>)</span><br><span class="line">time.Sleep(time.Second*<span class="number">5</span>)</span><br><span class="line">log.Println(<span class="string">"client closed"</span>)</span><br><span class="line">}</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><h2 id="使用-Kqueue-实现简单的-TCP-服务器"><a href="#使用-Kqueue-实现简单的-TCP-服务器" class="headerlink" title="使用 Kqueue 实现简单的 TCP 服务器"></a>使用 Kqueue 实现简单的 TCP 服务器</h2><p><a href="https://github.com/FRosner/FrSrv">项目地址</a></p>
<h3 id="设计"><a href="#设计" class="headerlink" title="设计"></a>设计</h3><p>TCP 服务器由 TCP socket，来自客户端连接的 socket，kqueue 以及轮询 kqueue 的 event loop 组成。</p>
<p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211223235715574.png" alt="image-20211223235715574"></p></summary>
<category term="网络" scheme="https://cwww3.github.io/tags/%E7%BD%91%E7%BB%9C/"/>
</entry>
<entry>
<title>HTTPS</title>
<link href="https://cwww3.github.io/2021/11/27/HTTPS/"/>
<id>https://cwww3.github.io/2021/11/27/HTTPS/</id>
<published>2021-11-27T12:52:40.000Z</published>
<updated>2022-09-03T18:36:37.818Z</updated>
<content type="html"><![CDATA[<h2 id="HTTPS"><a href="#HTTPS" class="headerlink" title="HTTPS"></a>HTTPS</h2><p>HTTP 由于是明⽂传输，所以安全上存在以下三个⻛险：</p><p>窃听⻛险，⽐如通信链路上可以获取通信内容，⽤户号容易没。</p><p>篡改⻛险，⽐如强制植⼊垃圾⼴告，视觉污染，⽤户眼容易瞎。</p><p>冒充⻛险，⽐如冒充淘宝⽹站，⽤户钱容易没。</p><span id="more"></span><p>HTTP<strong>S</strong> 在 HTTP 与 TCP 层之间加⼊了 SSL/TLS 协议，可以很好的解决 HTTP 的安全问题。</p><ul><li><p>混合加密的⽅式实现信息的机密性，解决了窃听的⻛险。</p><p>在通信建⽴前采⽤⾮对称加密的⽅式交换「会话秘钥」，后续就不再使⽤⾮对称加密。</p><p>在通信过程中全部使⽤对称加密的「会话秘钥」的⽅式加密明⽂数据。</p><p>对称加密只使⽤⼀个密钥，运算速度快，密钥必须保密，⽆法做到安全的密钥交换。⾮对称加密使⽤两个密钥：公钥和私钥，公钥可以任意分发⽽私钥保密，解决了密钥交换问题但速度慢。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127200955709.png" alt="image-20211127200955709" style="zoom:25%;" /></li><li><p>摘要算法的⽅式来实现完整性，它能够为数据⽣成独⼀⽆⼆的「指纹」，指纹⽤于校验数据的完整性，解决了篡改的⻛险。</p><p>客户端在发送明⽂之前会通过摘要算法算出明⽂的「指纹」，发送的时候把「指纹 + 明⽂」⼀同加密成密⽂后，发送给服务器，服务器解密后，⽤相同的摘要算法算出发送过来的明⽂，通过⽐较客户端携带的「指纹」和当前算出的「指纹」做⽐较，若「指纹」相同，说明数据是完整的。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127201037389.png" alt="image-20211127201037389" style="zoom:25%;" /></li><li><p>数字证书，解决了冒充的⻛险。</p><p>客户端先向服务器端索要公钥，然后⽤公钥加密信息，服务器收到密⽂后，⽤⾃⼰的私钥解密。这就存在些问题，如何保证公钥不被篡改和信任度？所以这⾥就需要借助第三⽅权威机构 CA （数字证书认证机构），将服务器公钥放在数字证书（由数字证书认证机构颁发）中，只要证书是可信的，公钥就是可信的。</p></li></ul><p> <img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127201414160.png" alt="image-20211127201414160" style="zoom:33%;" /></p><h3 id="SSL-TLS-协议基本流程"><a href="#SSL-TLS-协议基本流程" class="headerlink" title="SSL/TLS 协议基本流程"></a>SSL/TLS 协议基本流程</h3><p>TSL 四次握手</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127204231432.png" alt="image-20211127204231432" style="zoom:50%;" /><ul><li>ClientHello</li></ul><p>⾸先，由客户端向服务器发起加密通信请求，也就是 ClientHello 请求。</p><p>在这⼀步，客户端主要向服务器发送以下信息：</p><p>（1）客户端⽀持的 SSL/TLS 协议版本，如 TLS 1.2 版本。</p><p>（2）客户端⽣产的随机数（ Client Random ），后⾯⽤于⽣产「会话秘钥」。</p><p>（3）客户端⽀持的密码套件列表，如 RSA 加密算法。</p><ul><li>ServerHello</li></ul><p>服务器收到客户端请求后，向客户端发出响应，也就是 SeverHello 。服务器回应的内容有如下内容：</p><p>（1）确认 SSL/ TLS 协议版本，如果浏览器不⽀持，则关闭加密通信。</p><p>（2）服务器⽣产的随机数（ Server Random ），后⾯⽤于⽣产「会话秘钥」。</p><p>（3）确认的密码套件列表，如 RSA 加密算法。</p><p>（4）服务器的数字证书。</p><ul><li>客户端回应</li></ul><p>客户端收到服务器的回应之后，⾸先通过浏览器或者操作系统中的 CA 公钥，确认服务器的数字证书的真实性。如果证书没有问题，客户端会从数字证书中取出服务器的公钥，然后使⽤它加密报⽂，向服务器发送如下信息：</p><p>（1）⼀个随机数（ pre-master key ）。该随机数会被服务器公钥加密。</p><p>（2）加密通信算法改变通知，表示随后的信息都将⽤「会话秘钥」加密通信。</p><p>（3）客户端握⼿结束通知，表示客户端的握⼿阶段已经结束。这⼀项同时把之前所有内容的发⽣的数据做个摘要，⽤来供服务端校验。上⾯第⼀项的随机数是整个握⼿阶段的第三个随机数，这样服务器和客户端就同时有三个随机数，接着就⽤双⽅协商的加密算法，各⾃⽣成本次通信的「会话秘钥」。</p><ul><li>服务器的最后回应</li></ul><p>服务器收到客户端的第三个随机数（ pre-master key ）之后，通过协商的加密算法，计算出本次通信的「会话秘钥」。然后，向客户端发⽣最后的信息：</p><p>（1）加密通信算法改变通知，表示随后的信息都将⽤「会话秘钥」加密通信。</p><p>（2）服务器握⼿结束通知，表示服务器的握⼿阶段已经结束。这⼀项同时把之前所有内容的发⽣的数据做个摘要，⽤来供客户端校验。</p><p>⾄此，整个 SSL/TLS 的握⼿阶段全部结束。接下来，客户端与服务器进⼊加密通信，就完全是使⽤普通的 HTTP 协议，只不过⽤「会话秘钥」加密内容。</p><p>事实上，不同的密钥交换算法，TLS 的握⼿过程可能会有⼀些区别。</p><p>这⾥先简单介绍下密钥交换算法，因为考虑到性能的问题，所以双⽅在加密应⽤信息时使⽤的是对称加密密钥，⽽对称加密密钥是不能被泄漏的，为了保证对称加密密钥的安全性，所以使⽤⾮对称加密的⽅式来保护对称加密密钥的协商，这个⼯作就是密钥交换算法负责的。接下来，我们就以最简单的 RSA 密钥交换算法，来看看它的 TLS 握⼿过程。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/16380164319252.png" alt="16380164319252" /><p><strong>RSA</strong> 算法的缺陷</p><p>使⽤ <strong>RSA</strong> 密钥协商算法的最⼤问题是不⽀持<strong>前向保密</strong>。因为客户端传递随机数（⽤于⽣成对称加密密钥的条件之⼀）给服务端时使⽤的是公钥加密的，服务端收到到后，会⽤私钥解密得到随机数。所以⼀旦服务端的私钥泄漏了，过去被第三⽅截获的所有 TLS 通讯密⽂都会被破解。</p><p>为了解决这⼀问题，于是就有了 DH 密钥协商算法，这⾥简单介绍它的⼯作流程。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127204841502.png" alt="image-20211127204841502" style="zoom:50%;" /><p>客户端和服务端各⾃会⽣成随机数，并以此作为私钥，然后根据公开的 DH 计算公示算出各⾃的公钥，通过 TLS 握⼿双⽅交换各⾃的公钥，这样双⽅都有⾃⼰的私钥和对⽅的公钥，然后双⽅根据各⾃持有的材料算出⼀个随机数，这个随机数的值双⽅都是⼀样的，这就可以作为后续对称加密时使⽤的密钥。</p><p>DH 密钥交换过程中，即使第三⽅截获了 <strong>TLS</strong> 握⼿阶段传递的公钥，在不知道的私钥的情况下，也是⽆法计算出密钥的，⽽且每⼀次对称加密密钥都是实时⽣成的，实现前向保密。但因为 DH 算法的计算效率问题，后⾯出现了 ECDHE 密钥协商算法，我们现在⼤多数⽹站使⽤的正是 ECDHE 密钥协商算法。</p>]]></content>
<summary type="html"><h2 id="HTTPS"><a href="#HTTPS" class="headerlink" title="HTTPS"></a>HTTPS</h2><p>HTTP 由于是明⽂传输，所以安全上存在以下三个⻛险：</p>
<p>窃听⻛险，⽐如通信链路上可以获取通信内容，⽤户号容易没。</p>
<p>篡改⻛险，⽐如强制植⼊垃圾⼴告，视觉污染，⽤户眼容易瞎。</p>
<p>冒充⻛险，⽐如冒充淘宝⽹站，⽤户钱容易没。</p></summary>
<category term="网络" scheme="https://cwww3.github.io/tags/%E7%BD%91%E7%BB%9C/"/>
</entry>
<entry>
<title>Nginx</title>
<link href="https://cwww3.github.io/2021/11/27/Nginx/"/>
<id>https://cwww3.github.io/2021/11/27/Nginx/</id>
<published>2021-11-27T10:00:07.000Z</published>
<updated>2021-11-27T12:02:32.533Z</updated>
<content type="html"><![CDATA[<h2 id="Nginx"><a href="#Nginx" class="headerlink" title="Nginx"></a>Nginx</h2><h3 id="二进制安装"><a href="#二进制安装" class="headerlink" title="二进制安装"></a>二进制安装</h3><p><a href="http://nginx.org/en/download.html">安装地址</a></p><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 解压nginx压缩包并进入nginx-1.20.2</span></span><br><span class="line">tar -xf nginx-1.20.2.tar.gz && cd nginx-1.20.2</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 安装所需软件</span></span><br><span class="line">yum install -y pcre pcre-devel #支持nginx的正则</span><br><span class="line">yum install -y openssl openssl-devel #加密认证</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 执行./configure 配置环境 生成Makefile文件</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> prefix指定安装的路径</span></span><br><span class="line">./configure --user=nginx --group=nginx --prefix=/etc/nginx/ --with-http_stub_status_module --with-http_ssl_module --with-pcre</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 编译并且安装</span> </span><br><span class="line">make&&make install</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 配置PATH路径</span> </span><br><span class="line">export PAHT=$PATH:/etc/nginx/sbin</span><br></pre></td></tr></table></figure><span id="more"></span><h3 id="Nginx命令"><a href="#Nginx命令" class="headerlink" title="Nginx命令"></a>Nginx命令</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 显示nginx的版本号和编译信息</span></span><br><span class="line">nginx -V </span><br><span class="line"><span class="meta">#</span><span class="bash"> 检查默认配置文件 /etc/nginx/conf/nginx.conf</span></span><br><span class="line">nginx -t</span><br><span class="line"><span class="meta">#</span><span class="bash"> 检查指定配置文件</span></span><br><span class="line">nginx -t -c xxx.conf</span><br><span class="line"><span class="meta">#</span><span class="bash"> 启动</span></span><br><span class="line">nginx</span><br><span class="line"><span class="meta">#</span><span class="bash"> 重新加载配置文件</span></span><br><span class="line">nginx -s reload</span><br><span class="line"><span class="meta">#</span><span class="bash"> 关闭</span></span><br><span class="line">nginx -s quit</span><br></pre></td></tr></table></figure><h3 id="添加到Systemd中管理"><a href="#添加到Systemd中管理" class="headerlink" title="添加到Systemd中管理"></a>添加到Systemd中管理</h3><ul><li>Systemctl 介绍</li></ul><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 查看版本</span></span><br><span class="line">systemctl --version </span><br><span class="line"><span class="meta">#</span><span class="bash"> 列出所有可用单元（服务）</span></span><br><span class="line">systemctl list-unit-files</span><br><span class="line"><span class="meta">#</span><span class="bash"> 列出所有运行中的单元</span></span><br><span class="line">systemctl list-units</span><br><span class="line"><span class="meta">#</span><span class="bash"> 列出所有失败的单元</span></span><br><span class="line">systemctl --failed</span><br><span class="line"><span class="meta">#</span><span class="bash"> 查看自启动的软件</span></span><br><span class="line">systemctl list-unit-files | grep enable</span><br><span class="line"><span class="meta">#</span><span class="bash"> 修改了配置文件后，重载配置文件</span></span><br><span class="line">systemctl daemon-reload</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 查看nginx是否开机启动</span></span><br><span class="line">systemctl is-enabled nginx.service</span><br><span class="line"><span class="meta">#</span><span class="bash"> 查看nginx状态</span></span><br><span class="line">systemctl status nginx.service</span><br><span class="line"><span class="meta">#</span><span class="bash"> 启动nginx单元</span></span><br><span class="line">systemctl start nginx.service，</span><br><span class="line"><span class="meta">#</span><span class="bash"> 重启nginx单元</span></span><br><span class="line">systemctl restart nginx.service</span><br><span class="line"><span class="meta">#</span><span class="bash"> 停止nginx单元</span></span><br><span class="line">systemctl stop nginx.service</span><br><span class="line"><span class="meta">#</span><span class="bash"> 重载nginx配置</span></span><br><span class="line">systemctl reload nginx.service，</span><br><span class="line"><span class="meta">#</span><span class="bash"> 设置开机自启动。</span></span><br><span class="line">systemctl enable nginx.service</span><br><span class="line"><span class="meta">#</span><span class="bash"> 关闭开机自启动</span></span><br><span class="line">systemctl disable nginx.service</span><br></pre></td></tr></table></figure><ul><li>将nginx加入管理</li></ul><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 在系统服务目录里创建nginx.service文件</span></span><br><span class="line">vim /usr/lib/systemd/system/nginx.service</span><br><span class="line"><span class="meta">#</span><span class="bash"> 写入</span></span><br><span class="line">[Unit] # 服务的说明</span><br><span class="line">Description=nginx # 描述服务</span><br><span class="line">After=network.target # 描述服务类别</span><br><span class="line"></span><br><span class="line">[Service] # 服务运行参数的设置</span><br><span class="line">Type=forking # 后台运行的形式</span><br><span class="line"><span class="meta">#</span><span class="bash"> [Service]的启动、重启、停止命令全部要求使用绝对路径</span></span><br><span class="line">ExecStart=/etc/nginx/sbin/nginx # 服务的具体运行命令</span><br><span class="line">ExecReload=/etc/nginx/sbin/nginx -s reload # 重启命令</span><br><span class="line">ExecStop=/etc/nginx/sbin/nginx -s quit # 停止命令</span><br><span class="line">PrivateTmp=true # PrivateTmp=True表示给服务分配独立的临时空间</span><br><span class="line"></span><br><span class="line">[Install] # [Install]运行级别下服务安装的相关设置，可设置为多用户，即系统运行级别为3</span><br><span class="line">WantedBy=multi-user.target</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 加载配置</span></span><br><span class="line">systemctl daemon-reload</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 启动nginx</span></span><br><span class="line">systemctl start nginx</span><br><span class="line"><span class="meta">#</span><span class="bash"> 开机自启动nginx</span></span><br><span class="line">systemctl enable nginx</span><br></pre></td></tr></table></figure><h3 id="添加Nginx模块"><a href="#添加Nginx模块" class="headerlink" title="添加Nginx模块"></a>添加Nginx模块</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 创建并进入目录</span></span><br><span class="line">madir -p /data/software && cd /data/software</span><br><span class="line"><span class="meta">#</span><span class="bash"> 下载</span></span><br><span class="line">git clone git@github.com:arut/nginx-rtmp-module.git</span><br><span class="line"><span class="meta">#</span><span class="bash"> 查看 nginx编译信息</span></span><br><span class="line">nginx -V </span><br><span class="line"></span><br><span class="line">nginx version: nginx/1.20.2</span><br><span class="line">built by gcc 8.5.0 20210514 (Red Hat 8.5.0-4) (GCC)</span><br><span class="line">built with OpenSSL 1.1.1k FIPS 25 Mar 2021</span><br><span class="line">TLS SNI support enabled</span><br><span class="line">configure arguments: --user=nginx --group=nginx --prefix=/etc/nginx/ --with-http_stub_status_module --with-http_ssl_module --with-pcre</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 停止nginx</span></span><br><span class="line">systemctl stop nginx</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 进入nginx安装目录</span></span><br><span class="line">cd /usr/local/src/nginx-1.20.2</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 执行 通过 --add-module添加刚才git下载的模块</span></span><br><span class="line">./configure --user=nginx --group=nginx --prefix=/etc/nginx/ --with-http_stub_status_module --with-http_ssl_module --with-pcre --add-module=/data/software/nginx-rtmp-module</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 执行make</span></span><br><span class="line">make # 不需要执行make install不然会覆盖</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 替换nginx二进制文件</span></span><br><span class="line">cp /etc/nginx/sbin/nginx /etc/nginx/sbin/nginx.bak</span><br><span class="line">rm /etc/nginx/sbin/nginx</span><br><span class="line">cp ./objs/nginx /etc/nginx/sbin/nginx</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 执行nginx -V 查看配置</span></span><br><span class="line">nginx version: nginx/1.20.2</span><br><span class="line">built by gcc 8.5.0 20210514 (Red Hat 8.5.0-4) (GCC)</span><br><span class="line">built with OpenSSL 1.1.1k FIPS 25 Mar 2021</span><br><span class="line">TLS SNI support enabled</span><br><span class="line">configure arguments: --user=nginx --group=nginx --prefix=/etc/nginx/ --with-http_stub_status_module --with-http_ssl_module --with-pcre --add-module=/data/software/nginx-rtmp-module</span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><h2 id="Nginx"><a href="#Nginx" class="headerlink" title="Nginx"></a>Nginx</h2><h3 id="二进制安装"><a href="#二进制安装" class="headerlink" title="二进制安装"></a>二进制安装</h3><p><a href="http://nginx.org/en/download.html">安装地址</a></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 解压nginx压缩包并进入nginx-1.20.2</span></span><br><span class="line">tar -xf nginx-1.20.2.tar.gz &amp;&amp; cd nginx-1.20.2</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 安装所需软件</span></span><br><span class="line">yum install -y pcre pcre-devel #支持nginx的正则</span><br><span class="line">yum install -y openssl openssl-devel #加密认证</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 执行./configure 配置环境 生成Makefile文件</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> prefix指定安装的路径</span></span><br><span class="line">./configure --user=nginx --group=nginx --prefix=/etc/nginx/ --with-http_stub_status_module --with-http_ssl_module --with-pcre</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 编译并且安装</span> </span><br><span class="line">make&amp;&amp;make install</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 配置PATH路径</span> </span><br><span class="line">export PAHT=$PATH:/etc/nginx/sbin</span><br></pre></td></tr></table></figure></summary>
<category term="Nginx" scheme="https://cwww3.github.io/tags/Nginx/"/>
</entry>
<entry>
<title>HTTP</title>
<link href="https://cwww3.github.io/2021/11/27/HTTP/"/>
<id>https://cwww3.github.io/2021/11/27/HTTP/</id>
<published>2021-11-27T04:03:04.000Z</published>
<updated>2022-09-03T18:36:37.819Z</updated>
<content type="html"><![CDATA[<h2 id="HTTP"><a href="#HTTP" class="headerlink" title="HTTP"></a>HTTP</h2><p>HTTP （HyperText Transfer Protocol）超文本传输协议</p><p><strong>HTTP</strong> 是⼀个在计算机世界⾥专⻔在「两点」之间「传输」⽂字、图⽚、⾳频、视频等「超⽂本」数据的「约定和规范」。</p><span id="more"></span><h3 id="HTTP-状态码"><a href="#HTTP-状态码" class="headerlink" title="HTTP 状态码"></a>HTTP 状态码</h3><ul><li>1xx</li></ul><p>1xx 类状态码属于提示信息，是协议处理中的⼀种中间状态，实际⽤到的⽐较少。</p><ul><li>2xx</li></ul><p>2xx 类状态码表示服务器成功处理了客户端的请求，也是我们最愿意看到的状态。</p><p>「<strong>200 OK</strong>」是最常⻅的成功状态码，表示⼀切正常。如果是⾮ HEAD 请求，服务器返回的响应头都会有 body 数据。</p><p>「<strong>204 No Content</strong>」也是常⻅的成功状态码，与 200 OK 基本相同，但响应头没有 body 数据。「<strong>206 Partial Content</strong>」是应⽤于 HTTP 分块下载或断点续传，表示响应返回的 body 数据并不是资源的全部，⽽是其中的⼀部分，也是服务器处理成功的状态。</p><ul><li>3xx</li></ul><p>3xx 类状态码表示客户端请求的资源发送了变动，需要客户端⽤新的 URL 重新发送请求获取资源，也就是重定向。</p><p>「<strong>301 Moved Permanently</strong>」表示永久重定向，说明请求的资源已经不存在了，需改⽤新的 URL 再次访问。</p><p>「<strong>302 Found</strong>」表示临时重定向，说明请求的资源还在，但暂时需要⽤另⼀个 URL 来访问。</p><p>301 和 302 都会在响应头⾥使⽤字段 Location ，指明后续要跳转的 URL，浏览器会⾃动重定向新的 URL。</p><p>「<strong>304 Not Modified</strong>」不具有跳转的含义，表示资源未修改，重定向已存在的缓冲⽂件，也称缓存重定向，⽤于缓存控制。</p><ul><li>4xx</li></ul><p>4xx 类状态码表示客户端发送的报⽂有误，服务器⽆法处理，也就是错误码的含义。</p><p>「<strong>400 Bad Request</strong>」表示客户端请求的报⽂有错误，但只是个笼统的错误。</p><p>「<strong>401 Unauthorized</strong>」表示服务器需要客户端提供认证信息。</p><p>「<strong>403 Forbidden</strong>」表示服务器禁⽌访问资源，并不是客户端的请求出错。</p><p>「<strong>404 Not Found</strong>」表示请求的资源在服务器上不存在或未找到，所以⽆法提供给客户端。</p><ul><li><em>5xx</em></li></ul><p>5xx 类状态码表示客户端请求报⽂正确，但是服务器处理时内部发⽣了错误，属于服务器端的错误码。</p><p>「<strong>500 Internal Server Error</strong>」与 400 类型，是个笼统通⽤的错误码，服务器发⽣了什么错误，我们并不知道。</p><p>「<strong>501 Not Implemented</strong>」表示客户端请求的功能还不⽀持，类似“即将开业，敬请期待”的意思。</p><p>「<strong>502 Bad Gateway</strong>」通常是<strong>服务器作为⽹关或代理时</strong>返回的错误码，表示服务器⾃身⼯作正常，访问后端服务器发⽣了错误。</p><p>「<strong>503 Service Unavailable</strong>」表示服务器当前很忙，暂时⽆法响应服务器，类似“⽹络服务正忙，请稍后重试”的意思。</p><h3 id="HTTP-常⻅字段"><a href="#HTTP-常⻅字段" class="headerlink" title="HTTP 常⻅字段"></a>HTTP 常⻅字段</h3><ul><li>HOST</li></ul><p>客户端发送请求时，⽤来指定服务器的域名。</p><ul><li><em>Content-Length</em></li></ul><p>服务器在返回数据时，会有 Content-Length 字段，表明本次回应的数据⻓度。</p><ul><li><em>Connection</em></li></ul><p>Connection 字段最常⽤于客户端要求服务器使⽤ TCP 持久连接，以便其他请求复⽤。</p><p>HTTP/1.1 版本的默认连接都是持久连接，但为了兼容⽼版本的 HTTP，需要指定 Connection ⾸部字段的值为 Keep-Alive 。</p><ul><li>Accept / <em>Content-Type</em></li></ul><p>客户端请求的时候，可以使⽤ Accept 字段声明⾃⼰可以接受哪些数据格式。</p><p>Content-Type 字段⽤于服务器回应时，告诉客户端，本次数据是什么格式。</p><ul><li>Accept-Encoding / Content-Encoding</li></ul><p>Content-Encoding 字段说明数据的压缩⽅法。表示服务器返回的数据使⽤了什么压缩格式</p><p>客户端在请求时，⽤ Accept-Encoding 字段说明⾃⼰可以接受哪些压缩⽅法。</p><h3 id="HTTP-特性"><a href="#HTTP-特性" class="headerlink" title="HTTP 特性"></a><strong>HTTP</strong> 特性</h3><ul><li>简单</li></ul><p>HTTP 基本的报⽂格式就是 header + body ，头部信息也是 key-value 简单⽂本的形式，易于理解，降低了学习和使⽤的⻔槛。</p><ul><li>灵活和易于扩展</li></ul><p>HTTP 协议⾥的各类请求⽅法、URI/URL、状态码、头字段等每个组成要求都没有被固定死，都允许开发⼈员⾃定义和扩充。</p><ul><li>无状态</li></ul><p>⽆状态的好处，因为服务器不会去记忆 HTTP 的状态，所以不需要额外的资源来记录状态信息，这能减轻服务器的负担，能够把更多的 CPU 和内存⽤来对外提供服务。</p><p>⽆状态的坏处，既然服务器没有记忆能⼒，它在完成有关联性的操作时会⾮常麻烦。例如登录->添加购物⻋->下单->结算->⽀付，这系列操作都要知道⽤户的身份才⾏。但服务器不知道这些请求是有关联的，每次都要问⼀遍身份信息。</p><p>对于⽆状态的问题，解法⽅案有很多种，其中⽐较简单的⽅式⽤ <strong>Cookie</strong> 技术。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127124400027.png" alt="image-20211127124400027" style="zoom:33%;" /><ul><li>明⽂传输</li></ul><p>明⽂意味着在传输过程中的信息，是可⽅便阅读的，通过浏览器的 F12 控制台或 Wireshark 抓包都可以直接⾁眼查看，为我们调试⼯作带了极⼤的便利性。</p><ul><li>不安全</li></ul><p>通信使⽤明⽂（不加密），内容可能会被窃听。不验证通信⽅的身份，因此有可能遭遇伪装。⽆法证明报⽂的完整性，所以有可能已遭篡改。可以通过引⼊ SSL/TLS 层，使得在安全上达到了极致。</p><h3 id="HTTP-性能"><a href="#HTTP-性能" class="headerlink" title="HTTP 性能"></a>HTTP 性能</h3><p>HTTP 协议是基于 <strong>TCP/IP</strong>，并且使⽤了「请求 <strong>-</strong> 应答」的通信模式，所以性能的关键就在这两点⾥。</p><ul><li>长连接</li></ul><p>早期 HTTP/1.0 性能上的⼀个很⼤的问题，那就是每发起⼀个请求，都要新建⼀次 TCP 连接（三次握⼿），⽽且是串⾏请求，做了⽆谓的 TCP 连接建⽴和断开，增加了通信开销。为了解决上述 TCP 连接问题，HTTP/1.1 提出了⻓连接的通信⽅式，也叫持久连接。这种⽅式的好处在于减少了 TCP 连接的重复建⽴和断开所造成的额外开销，减轻了服务器端的负载。持久连接的特点是，只要任意⼀端没有明确提出断开连接，则保持 TCP 连接状态。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211127111558513.png" alt="image-20211127111558513" style="zoom:33%;" /><ul><li>管道传输</li></ul><p>可在同⼀个 TCP 连接⾥⾯，客户端可以发起多个请求，只要第⼀个请求发出去了，不必等其回来，就可以发第⼆个请求出去，可以减少整体的响应时间。但是服务器还是按照顺序，先回应 A 请求，完成后再回应 B 请求。要是前⾯的回应特别慢，后⾯就会有许多请求排队等着。这称为「队头堵塞」。</p><h3 id="优化方案"><a href="#优化方案" class="headerlink" title="优化方案"></a>优化方案</h3><h4 id="避免发送-重复的HTTP-请求"><a href="#避免发送-重复的HTTP-请求" class="headerlink" title="避免发送 重复的HTTP 请求"></a>避免发送 重复的<strong>HTTP</strong> 请求</h4><p>通过<strong>缓存技术</strong>，对于⼀些具有重复性的 HTTP 请求，⽐如每次请求得到的数据都⼀样的，我们可以把这对「请求-响应」的数据都缓存在本地，那么下次就直接读取本地的数据，不必在通过⽹络获取服务器的响应了。</p><p>客户端会把第⼀次请求以及响应的数据保存在本地磁盘上，其中将请求的 URL 作为 key，⽽响应作为 value，两者形成映射关系。这样当后续发起相同的请求时，就可以先在本地磁盘上通过 key 查到对应的 value，也就是响应，如果找到了，就直接从本地读取该响应。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211127113034457.png" alt="image-20211127113034457" style="zoom:33%;" /><p>万⼀缓存的响应不是最新的，⽽客户端并不知情，<strong>那么该怎么办呢</strong>？</p><p>服务器在发送 HTTP 响应时，会估算⼀个过期的时间，并把这个信息放到响应头部中，这样客户端在查看响应头部的信息时，⼀旦发现缓存的响应是过期的，则就会重新发送⽹络请求。</p><p>如果客户端从第⼀次请求得到的响应头部中发现该响应过期了，客户端重新发送请求，假设服务器上的资源并没有变更，还是⽼样⼦，那么服务器的响应应该带上这个资源吗？很显然不带的话，可以提⾼ HTTP 协议的性能，<strong>那具体如何做到呢</strong>？</p><p>只需要客户端在重新发送请求时，在请求的 Etag 头部带上第⼀次请求的响应头部中的摘要，这个摘要是唯⼀标识响应的资源，当服务器收到请求后，会将本地资源的摘要与请求中的摘要做个⽐较。如果不同，那么说明客户端的缓存已经没有价值，服务器在响应中带上最新的资源。如果相同，说明客户端的缓存还是可以继续使⽤的，那么服务器仅返回不含有包体的 304 Not Modified 304 Not Modified 响应，告诉客户端仍然有效，这样就可以减少响应资源在⽹络中传输的延时。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/pic/image-20211127124511250.png" alt="image-20211127124511250" style="zoom:25%;" /><h4 id="减少-HTTP-请求次数"><a href="#减少-HTTP-请求次数" class="headerlink" title="减少 HTTP 请求次数"></a>减少 <strong>HTTP</strong> 请求次数</h4><ul><li>减少重定向请求次数</li></ul><p>服务器上的⼀个资源可能由于迁移、维护等原因从 url1 移⾄ url2 后，⽽客户端不知情，它还是继续请求 url1，这时服务器不能粗暴地返回错误，⽽是通过 302 响应码和 Location 头部，告诉客户端该资源已经迁移⾄ url2 了，于是客户端需要再发送 url2 请求以获得服务器的资源。</p><p>如果重定向请求越多，那么客户端就要多次发起 HTTP 请求，每⼀次的 HTTP 请求都得经过⽹络，这⽆疑会越降低⽹络性能。</p><p>另外，服务端这⼀⽅往往不只有⼀台服务器，⽐如源服务器上⼀级是代理服务器，然后代理服务器才与客户端通信，这时客户端重定向就会导致客户端与代理服务器之间需要 2 次消息传递。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211127114016498.png" alt="image-20211127114016498" style="zoom:33%;" /><p>如果重定向的⼯作交由代理服务器完成，就能减少 <strong>HTTP</strong> 请求次数了</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211127114045230.png" alt="image-20211127114045230" style="zoom: 33%;" /><p>⽽且当代理服务器知晓了重定向规则后，可以进⼀步减少消息传递次数</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211127114147959.png" alt="image-20211127114147959" style="zoom:33%;" /><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211127114329172.png" alt="image-20211127114329172" style="zoom:33%;" /><p>其中， 301 和 308 响应码是告诉客户端可以将重定向响应缓存到本地磁盘，之后客户端就⾃动⽤ url2 替代 url1 访问服务器的资源。</p><ul><li>合并请求</li></ul><p>如果把多个访问⼩⽂件的请求合并成⼀个⼤的请求，虽然传输的总资源还是⼀样，但是减少请求，也就意味着减少了重复发送的 <strong>HTTP</strong> 头部。</p><p>另外由于 HTTP/1.1 是请求响应模型，如果第⼀个发送的请求，未收到对应的响应，那么后续的请求就不会发送，于是为了防⽌单个请求的阻塞，所以⼀般浏览器会同时发起 5-6 个请求，每⼀个请求都是不同的 TCP 连接，那么如果合并了请求，也就会减少 <strong>TCP</strong> 连接的数量，因⽽省去了 <strong>TCP</strong> 握⼿和慢启动过程耗费的时间。</p><p>有的⽹⻚会含有很多⼩图⽚、⼩图标，有多少个⼩图⽚，客户端就要发起多少次请求。那么对于这些⼩图⽚，我们可以考虑使⽤ CSS Image Sprites 技术把它们合成⼀个⼤图⽚，这样浏览器就可以⽤⼀次请求获得⼀个⼤图⽚，然后再根据 CSS 数据把⼤图⽚切割成多张⼩图⽚。</p><p>除了将⼩图⽚合并成⼤图⽚的⽅式，还有服务端使⽤ webpack 等打包⼯具将 js、css 等资源合并打包成⼤⽂件，也是能达到类似的效果。</p><p>还可以将图⽚的⼆进制数据⽤ base64 编码后，以 URL 的形式潜⼊到 HTML ⽂件，跟随 HTML ⽂件⼀并发送.</p><figure class="highlight html"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag"><<span class="name">image</span></span></span><br><span class="line"><span class="tag"><span class="attr">src</span>=<span class="string">"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAPoAAAFKCAIAAAC7M9WrAAAACXBIWXMAA</span></span></span><br><span class="line"><span class="string"><span class="tag">... /></span></span></span><br></pre></td></tr></table></figure><p>但是这样的合并请求会带来新的问题，当⼤资源中的某⼀个⼩资源发⽣变化后，客户端必须重新下载整个完整的⼤资源⽂件，这显然带来了额外的⽹络消耗。</p><ul><li>延迟发送请求</li></ul><p>请求⽹⻚的时候，没必要把全部资源都获取到，⽽是只获取当前⽤户所看到的⻚⾯资源，当⽤户向下滑动⻚⾯的时候，再向服务器获取接下来的资源，这样就达到了延迟发送请求的效果。</p><h4 id="减少-HTTP-响应的数据⼤⼩"><a href="#减少-HTTP-响应的数据⼤⼩" class="headerlink" title="减少 HTTP 响应的数据⼤⼩"></a>减少 <strong>HTTP</strong> 响应的数据⼤⼩</h4><p>对于 HTTP 的请求和响应，通常 HTTP 的响应的数据⼤⼩会⽐较⼤，也就是服务器返回的资源会⽐较⼤。</p><p>于是，我们可以考虑对响应的资源进⾏压缩，这样就可以减少响应的数据⼤⼩，从⽽提⾼⽹络传输的效率。压缩的⽅式⼀般分为 2 种:</p><ul><li>⽆损压缩</li></ul><p>⽆损压缩是指资源经过压缩后，信息不被破坏，还能完全恢复到压缩前的原样，适合⽤在⽂本⽂件、程序可执⾏⽂件、程序源代码。</p><p>⾸先，我们针对代码的语法规则进⾏压缩，因为通常代码⽂件都有很多换⾏符或者空格，这些是为了帮助程序员更好的阅读，但是机器执⾏时并不要这些符，把这些多余的符号给去除掉。</p><p>接下来，就是⽆损压缩了，需要对原始资源建⽴统计模型，利⽤这个统计模型，将常出现的数据⽤较短的⼆进制⽐特序列表示，将不常出现的数据⽤较⻓的⼆进制⽐特序列表示，⽣成⼆进制⽐特序列⼀般是「霍夫曼编码」算法。</p><p>gzip 就是⽐较常⻅的⽆损压缩。客户端⽀持的压缩算法，会在 HTTP 请求中通过头部中的 Accept-Encoding 字段。</p><p>gzip 的压缩效率相⽐ Google 推出的 Brotli 算法还是差点意思，所以如果可以，服务器应该选择压缩效率更⾼的 br 压缩算法。</p><ul><li>有损压缩</li></ul><p>与⽆损压缩相对的就是有损压缩，经过此⽅法压缩，解压的数据会与原始数据不同但是⾮常接近。</p><p>有损压缩主要将次要的数据舍弃，牺牲⼀些质 ᰁ 来减少数据 ᰁ、提⾼压缩⽐，这种⽅法经常⽤于压缩多媒体数据，⽐如⾳频、视频、图⽚。</p><p>可以通过 HTTP 请求头部中的 Accept 字段⾥的「 q 质量因⼦」，告诉服务器期望的资源质量</p><figure class="highlight http"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="attribute">Accept</span><span class="punctuation">: </span>audio/*; q=0.2, audio/basic</span><br></pre></td></tr></table></figure><p>关于图⽚的压缩，⽬前压缩⽐较⾼的是 Google 推出的 <strong>WebP</strong> 格式，相同质量的图片下，WebP 格式的图⽚⼤⼩都⽐ Png 格式的图⽚⼩。</p><p>关于⾳视频的压缩，⾳视频主要是动态的，每个帧都有时序的关系，通常时间连续的帧之间的变化是很⼩的。</p><p>⽐如，⼀个在看书的视频，画⾯通常只有⼈物的⼿和书桌上的书是会有变化的，⽽其他地⽅通常都是静态的，于是只需要在⼀个静态的关键帧，使⽤<strong>增量数据</strong>来表达后续的帧，这样便减少了很多数据，提⾼了⽹络传输的性能。对于视频常⻅的编码格式有 H264、H265 等，⾳频常⻅的编码格式有 AAC、AC3。</p>]]></content>
<summary type="html"><h2 id="HTTP"><a href="#HTTP" class="headerlink" title="HTTP"></a>HTTP</h2><p>HTTP （HyperText Transfer Protocol）超文本传输协议</p>
<p><strong>HTTP</strong> 是⼀个在计算机世界⾥专⻔在「两点」之间「传输」⽂字、图⽚、⾳频、视频等「超⽂本」数据的「约定和规范」。</p></summary>
<category term="网络" scheme="https://cwww3.github.io/tags/%E7%BD%91%E7%BB%9C/"/>
</entry>
<entry>
<title>IO</title>
<link href="https://cwww3.github.io/2021/11/18/IO/"/>
<id>https://cwww3.github.io/2021/11/18/IO/</id>
<published>2021-11-18T03:22:05.000Z</published>
<updated>2021-11-18T03:22:39.818Z</updated>
<content type="html"><![CDATA[<h2 id="I-O"><a href="#I-O" class="headerlink" title="I/O"></a>I/O</h2><h3 id="流"><a href="#流" class="headerlink" title="流"></a>流</h3><ul><li><p>可以进行I/O操作的内核对象</p></li><li><p>文件、管道、套接字等都是流</p></li><li><p>流的入口：文件描述符(fd)</p><span id="more"></span><h3 id="I-O操作"><a href="#I-O操作" class="headerlink" title="I/O操作"></a>I/O操作</h3></li><li><p>对流的读写操作称为I/O操作</p></li></ul><h3 id="同步异步"><a href="#同步异步" class="headerlink" title="同步异步"></a>同步异步</h3><ul><li>同步就是发起一个调用后，被调用者未处理完请求之前，调用不返回。</li><li>异步就是发一个调用后，立刻得到被调用者的回应表示已接收到请求，但是被调用者并没有返回结果，此时可以处理其他的请求，被调用者通常依靠事件、回调等机制来通知调用者其返回结果。</li></ul><h3 id="阻塞和非阻塞"><a href="#阻塞和非阻塞" class="headerlink" title="阻塞和非阻塞"></a>阻塞和非阻塞</h3><ul><li>阻塞就是发起一个请求，调用者一直等待请求结果返回，也就是当前线程会被挂起，无法从事其他任务，只有当条件就绪才能继续。不占用CPU资源。</li><li>非阻塞就是发起一个请求，调用者不用一直等着结果返回，可以先去干其他的事情。占用CPU资源(轮询)</li></ul><h3 id="同步异步与阻塞非阻塞"><a href="#同步异步与阻塞非阻塞" class="headerlink" title="同步异步与阻塞非阻塞"></a>同步异步与阻塞非阻塞</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 老张烧开水的故事</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 老张爱喝茶，废话不说，煮开水。</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 出场人物：老张，水壶两把（普通水壶，简称水壶；会响的水壶，简称响水壶）。</span></span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 同步阻塞</span></span><br><span class="line">老张把水壶放到火上，立等水开。</span><br><span class="line">老张觉得自己有点傻</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 同步非阻塞</span></span><br><span class="line">老张把水壶放到火上，去客厅看电视，时不时去厨房看看水开没有。</span><br><span class="line">老张还是觉得自己有点傻，于是变高端了，买了把会响笛的那种水壶。水开之后，能大声发出嘀~~的噪音。</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 异步阻塞</span></span><br><span class="line">老张把响水壶放到火上，立等水开。</span><br><span class="line">老张觉得这样傻等意义不大</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 异步非阻塞</span></span><br><span class="line">老张把响水壶放到火上，去客厅看电视，水壶响之前不再去看它了，响了再去拿壶。</span><br><span class="line">老张觉得自己聪明了。</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 所谓同步异步，只是对于水壶而言</span></span><br><span class="line">普通水壶：同步；响水壶：异步。</span><br><span class="line">虽然都能干活，但响水壶可以在自己完工之后，提示老张水开了，这是普通水壶所不能及的。</span><br><span class="line">同步只能让调用者去轮询自己（情况2中），造成老张效率的低下。</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 所谓阻塞非阻塞，仅仅对于老张而言</span></span><br><span class="line">立等的老张：阻塞；看电视的老张：非阻塞。</span><br><span class="line">虽然情况3中响水壶是异步的，可对于立等的老张没有太大的意义。所以一般异步是配合非阻塞使用的，这样才能发挥异步的效用。</span><br><span class="line"></span><br></pre></td></tr></table></figure><h3 id="阻塞IO-Blocking-IO"><a href="#阻塞IO-Blocking-IO" class="headerlink" title="阻塞IO - Blocking IO"></a>阻塞IO - Blocking IO</h3><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118105721114.png" alt="image-20211118105721114" style="zoom:33%;" /><ul><li>一请求一应答</li><li>通常由一个独立的 <code>Acceptor</code> 线程负责监听客户端的连接。我们一般通过在 <code>while(true)</code> 循环中服务端会调用 <code>accept()</code> 方法等待客户端连接的方式监听请求，请求一旦接收到一个连接请求，就可以建立通信套接字在这个通信套接字上进行读写操作，此时不能再接收其他客户端连接请求，只能等待当前连接的客户端的操作执行完成，不过可以<strong>通过多线程来支持多个客户端的连接</strong></li></ul><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118103446987.png" alt="image-20211118103446987" style="zoom: 33%;" /><h3 id="伪异步-I-O"><a href="#伪异步-I-O" class="headerlink" title="伪异步 I/O"></a>伪异步 I/O</h3><p>当客户端并发访问量增加后<strong>阻塞IO模型</strong>会出随着并发访问量增加会导致线程数急剧膨胀可能会导致线程堆栈溢出、创建新线程失败等问题，最终导致进程宕机或者僵死，不能对外提供服务。</p><p>线程是宝贵的资源。线程的创建和销毁成本很高，线程本身占用较大内存，线程的切换成本也很高，容易造成锯齿状的系统负载。</p><ul><li>解决方案</li></ul><p>后端通过一个线程池来处理多个客户端的请求接入，设置线程的最大值，防止由于海量并发接入导致线程耗尽。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118105303658.png" alt="image-20211118105303658" style="zoom:33%;" /><h3 id="非阻塞IO-NoneBlocking-IO"><a href="#非阻塞IO-NoneBlocking-IO" class="headerlink" title="非阻塞IO - NoneBlocking IO"></a>非阻塞IO - NoneBlocking IO</h3><p>当用户线程发起一个 IO 操作后，并不需要等待，而是马上就得到一个结果。如果结果是一个 error 时，它就知道数据还没有准备好，于是它可以再次发送 IO 操作。一旦内核中的数据准备好了，并且又再次收到了用户线程的请求，那么它马上就将数据拷贝到了用户线程，然后返回。</p><p>在非阻塞IO 模型中，用户线程需要不断地询问内核数据是否就绪，也就说非阻塞IO不会交出CPU，而会一直占用CPU。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118110037961.png" alt="image-20211118110037961" style="zoom:33%;" /><ul><li>非阻塞式主要体现在用户进程发起recvfrom系统调用的时候,这个时候系统内核还没有接收到数据报,直接返回错误给用户进程,告诉“当前还没有数据报可达,晚点再来”</li><li>用户进程接收到信息,但是用户进程不知道什么时候数据报可达,于是就开始不断轮询(polling)向系统内核发起recvfrom的系统调用“询问数据来了没”,如果没有则继续返回错误</li><li>用户进程轮询发起recvfrom系统调用直至数据报可达,这个时候需要等待系统内核复制数据报到用户进程的缓冲区,复制完成之后将返回成功提示</li></ul><h3 id="IO多路复用-IO-multiplexing"><a href="#IO多路复用-IO-multiplexing" class="headerlink" title="IO多路复用 - IO multiplexing"></a>IO多路复用 - IO multiplexing</h3><p>所谓 I/O 多路复用机制，就是说通过一种机制，可以监视多个描述符，一旦某个描述符就绪（一般是读就绪或写就绪），能够通知程序进行相应的读写操作。这种机制的使用需要 <code>select</code> 、 <code>poll</code> 、 <code>epoll</code> 来配合。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118110236775.png" alt="image-20211118110236775" style="zoom:33%;" /><p>在多路复用IO模型中，会有一个内核线程不断地去轮询多个 socket 的状态，只有当真正读写事件发送时，才真正调用实际的IO读写操作。<strong>因为在多路复用IO模型中，只需要使用一个线程就可以管理多个socket，系统不需要建立新的进程或者线程，也不必维护这些线程和进程，并且只有真正有读写事件进行时，才会使用IO资源，所以它大大减少来资源占用</strong>。</p><ul><li>IO复用模式是使用select或者poll函数向系统内核发起调用，阻塞在这两个系统函数调用，而不是真正阻塞于实际的IO操作(recvfrom调用才是实际阻塞IO操作的系统调用)</li><li>阻塞于select函数的调用,等待数据报套接字变为可读状态</li><li>当select套接字返回可读状态的时候,就可以发起recvfrom调用把数据报复制到用户空间的缓冲区</li></ul><h3 id="信号驱动IO-signal-driven-IO"><a href="#信号驱动IO-signal-driven-IO" class="headerlink" title="信号驱动IO - signal driven IO"></a>信号驱动IO - signal driven IO</h3><p>在信号驱动IO模型中，当用户线程发起一个IO请求操作，会给对应的socket注册一个信号函数，然后用户线程会继续执行，当内核数据就绪时会发送一个信号给用户线程，用户线程接收到信号后，便在信号函数中调用IO读写操作来进行实际的IO请求操作。这个一般用于UDP中，对TCP套接字几乎没用，原因是该信号产生得过于频繁，并且该信号的出现并没有告诉我们发生了什么请求。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118110908521.png" alt="image-20211118110908521" style="zoom:33%;" /><h3 id="异步IO-asynchronous-IO"><a href="#异步IO-asynchronous-IO" class="headerlink" title="异步IO - asynchronous IO"></a>异步IO - asynchronous IO</h3><p>前面四种IO模型实际上都属于同步IO，只有最后一种是真正的异步IO，因为无论是多路复用IO还是信号驱动模型，IO操作的第2个阶段都会引起用户线程阻塞，也就是从内核拷贝数据到用户态的过程都会让用户线程阻塞。</p><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118110819248.png" alt="image-20211118110819248" style="zoom:33%;" /><ul><li>由POSIX规范定义，告知系统内核启动某个操作，并让内核在整个操作包含数据等待以及数据复制过程的完成之后通知用户进程数据已经准备完成，可以进行读取数据；</li><li>与上述的信号IO模型区分在于异步是通知我们何时IO操作完成,而信号IO是通知我们何时可以启动一个IO操作</li></ul><h3 id="IO模型对比"><a href="#IO模型对比" class="headerlink" title="IO模型对比"></a>IO模型对比</h3><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118111021556.png" alt="image-20211118111021556" style="zoom:33%;" /><h2 id="处理多I-O请求"><a href="#处理多I-O请求" class="headerlink" title="处理多I/O请求"></a>处理多I/O请求</h2><p><strong>阻塞+多进程/多线程</strong></p><p><strong>非阻塞+轮询</strong></p><p><strong>多路复用</strong></p><ul><li>select 最多监听1024个 且不会通知具体哪个流接收到数据 需要遍历全部流，进行处理。 平台无关性。</li></ul><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> {</span><br><span class="line"> <span class="keyword">select</span>() <span class="comment">//阻塞</span></span><br><span class="line"> </span><br><span class="line"> <span class="keyword">for</span> _, in := ins {</span><br><span class="line"> <span class="keyword">if</span> has {</span><br><span class="line"> <span class="comment">// 处理</span></span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure><ul><li><p>poll本质上和select没有区别，它将用户传入的数组拷贝到内核空间，然后查询每个fd对应的设备状态， <strong>但是它没有最大连接数的限制</strong>，原因是它是基于链表来存储的.</p></li><li><p>epoll 监听的数量与操作系统能打开的文件数相同，且返回收到数据的流。不同于忙轮询和无差别轮询，epoll会把哪个流发生了怎样的I/O事件通知我们。Linux操作系统的方法。</p></li></ul><figure class="highlight go"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> {</span><br><span class="line"> ins = epoll() <span class="comment">//阻塞 </span></span><br><span class="line"> </span><br><span class="line"> <span class="keyword">for</span> _, in := ins {</span><br><span class="line"> <span class="comment">// 处理</span></span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure><ul><li><img src="https://raw.githubusercontent.com/cwww3/picture/master/img/image-20211118111811931.png" alt="image-20211118111811931" style="zoom:33%;" /></li></ul>]]></content>
<summary type="html"><h2 id="I-O"><a href="#I-O" class="headerlink" title="I/O"></a>I/O</h2><h3 id="流"><a href="#流" class="headerlink" title="流"></a>流</h3><ul>
<li><p>可以进行I/O操作的内核对象</p>
</li>
<li><p>文件、管道、套接字等都是流</p>
</li>
<li><p>流的入口：文件描述符(fd)</p></summary>
<category term="IO" scheme="https://cwww3.github.io/tags/IO/"/>
</entry>
<entry>
<title>Dockerfile</title>
<link href="https://cwww3.github.io/2021/11/05/docker-file/"/>
<id>https://cwww3.github.io/2021/11/05/docker-file/</id>
<published>2021-11-05T03:13:55.000Z</published>
<updated>2021-11-05T03:14:43.393Z</updated>
<content type="html"><![CDATA[<h3 id="Dockerfile"><a href="#Dockerfile" class="headerlink" title="Dockerfile"></a>Dockerfile</h3><ul><li>RUN</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">RUN</span><span class="bash"> apt-get update && apt-get install -y \</span></span><br><span class="line"><span class="bash"> aufs-tools \</span></span><br><span class="line"><span class="bash"> automake \</span></span><br><span class="line"><span class="bash"> build-essential \</span></span><br><span class="line"><span class="bash"> curl \</span></span><br><span class="line"><span class="bash"> dpkg-sig \</span></span><br><span class="line"><span class="bash"> libcap-dev \</span></span><br><span class="line"><span class="bash"> libsqlite3-dev \</span></span><br><span class="line"><span class="bash"> mercurial \</span></span><br><span class="line"><span class="bash"> reprepro \</span></span><br><span class="line"><span class="bash"> ruby1.9.1 \</span></span><br><span class="line"><span class="bash"> ruby1.9.1-dev \</span></span><br><span class="line"><span class="bash"> s3cmd=1.1.* \</span></span><br><span class="line"><span class="bash"> && rm -rf /var/lib/apt/lists/*</span></span><br></pre></td></tr></table></figure><span id="more"></span><ul><li>CMD</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 多数情况下，CMD 都需要一个交互式的 shell (bash, Python, perl 等)，</span></span><br><span class="line"><span class="comment"># 例如 CMD ["perl", "-de0"]，或者 CMD ["PHP", "-a"]。</span></span><br><span class="line"><span class="comment"># 使用这种形式意味着，当你执行类似docker run -it python时，你会进入一个准备好的 shell 中。</span></span><br></pre></td></tr></table></figure><ul><li>ENV</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 为了方便新程序运行，你可以使用ENV来为容器中安装的程序更新 PATH 环境变量。</span></span><br><span class="line"><span class="comment"># 例如使用ENV PATH /usr/local/nginx/bin:$PATH来确保CMD ["nginx"]能正确运行。</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">ENV</span> PG_MAJOR <span class="number">9.3</span></span><br><span class="line"><span class="keyword">ENV</span> PG_VERSION <span class="number">9.3</span>.<span class="number">4</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> curl -SL http://example.com/postgres-<span class="variable">$PG_VERSION</span>.tar.xz | tar -xJC /usr/src/postgress && …ENV PATH /usr/<span class="built_in">local</span>/postgres-<span class="variable">$PG_MAJOR</span>/bin:<span class="variable">$PATH</span></span></span><br></pre></td></tr></table></figure><ul><li>ADD COPY</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 一般优先使用 COPY。因为它比 ADD 更透明。</span></span><br><span class="line"><span class="comment"># ADD的最佳用例是将本地 tar 文件自动提取到镜像中</span></span><br><span class="line"><span class="keyword">ADD</span><span class="bash"> rootfs.tar.xz</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> mkdir -p /usr/src/things \</span></span><br><span class="line"><span class="bash"> && curl -SL http://example.com/big.tar.xz \</span></span><br><span class="line"><span class="bash"> | tar -xJC /usr/src/things \</span></span><br><span class="line"><span class="bash"> && make -C /usr/src/things all</span></span><br></pre></td></tr></table></figure><ul><li>ENTRYPOINT</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 最佳用处是设置镜像的主命令,允许将镜像当成命令本身来运行,用 CMD 提供默认选项</span></span><br><span class="line"><span class="keyword">ENTRYPOINT</span><span class="bash"> [<span class="string">"s3cmd"</span>] </span></span><br><span class="line"><span class="keyword">CMD</span><span class="bash"> [<span class="string">"--help"</span>]</span></span><br><span class="line"></span><br><span class="line">$docker <span class="keyword">run</span><span class="bash"> s3cmd</span></span><br><span class="line">$docker <span class="keyword">run</span><span class="bash"> s3cmd ls s3://mybucket</span></span><br></pre></td></tr></table></figure><ul><li>VOLUMN</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># VOLUME指令用于暴露任何数据库存储文件，配置文件，或容器创建的文件和目录。强烈建议使用 VOLUME来管理镜像中的可变部分和用户可以改变的部分。</span></span><br><span class="line"></span><br></pre></td></tr></table></figure><ul><li>USER</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 如果某个服务不需要特权执行，建议使用 USER 指令切换到非 root 用户。</span></span><br><span class="line"><span class="comment"># 如果要依赖确定的 UID/GID，你应该显示的指定一个 UID/GID</span></span><br><span class="line"><span class="keyword">RUN</span><span class="bash"> groupadd -r postgres && useradd -r -g postgres postgres </span></span><br></pre></td></tr></table></figure><ul><li>WORKDIR</li></ul><figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 为了清晰性和可靠性，你应该总是在WORKDIR中使用绝对路径 替代类似于 RUN cd </span></span><br></pre></td></tr></table></figure>]]></content>
<summary type="html"><h3 id="Dockerfile"><a href="#Dockerfile" class="headerlink" title="Dockerfile"></a>Dockerfile</h3><ul>
<li>RUN</li>
</ul>
<figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">RUN</span><span class="bash"> apt-get update &amp;&amp; apt-get install -y \</span></span><br><span class="line"><span class="bash"> aufs-tools \</span></span><br><span class="line"><span class="bash"> automake \</span></span><br><span class="line"><span class="bash"> build-essential \</span></span><br><span class="line"><span class="bash"> curl \</span></span><br><span class="line"><span class="bash"> dpkg-sig \</span></span><br><span class="line"><span class="bash"> libcap-dev \</span></span><br><span class="line"><span class="bash"> libsqlite3-dev \</span></span><br><span class="line"><span class="bash"> mercurial \</span></span><br><span class="line"><span class="bash"> reprepro \</span></span><br><span class="line"><span class="bash"> ruby1.9.1 \</span></span><br><span class="line"><span class="bash"> ruby1.9.1-dev \</span></span><br><span class="line"><span class="bash"> s3cmd=1.1.* \</span></span><br><span class="line"><span class="bash"> &amp;&amp; rm -rf /var/lib/apt/lists/*</span></span><br></pre></td></tr></table></figure></summary>
<category term="Docker" scheme="https://cwww3.github.io/tags/Docker/"/>
</entry>
</feed>