-
Notifications
You must be signed in to change notification settings - Fork 0
/
retry.go
199 lines (171 loc) · 4.1 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
package retry
import (
"context"
"errors"
"time"
)
// Policy is a retry policy.
type Policy struct {
// MinDelay is a first delay for retrying.
// Zero or negative value means no delay.
MinDelay time.Duration
// MaxDelay is the maximum delay for retrying.
// If MaxDelay is less than MinDelay, MinDelay is used as the maximum delay.
MaxDelay time.Duration
// MaxCount is max retry count.
// Zero or negative value means retry forever.
MaxCount int
// Jitter adds random delay.
// Zero means no jitter.
// Negative value shorten the delay.
Jitter time.Duration
}
// Retrier handles retrying.
type Retrier struct {
ctx context.Context
policy *Policy
count int
maxCount int
delay time.Duration
maxDelay time.Duration
timer *time.Timer
err error
}
// Start starts retrying
func (p *Policy) Start(ctx context.Context) *Retrier {
maxDelay := p.MaxDelay
if maxDelay < p.MinDelay {
maxDelay = p.MinDelay
}
return &Retrier{
ctx: ctx,
policy: p,
count: 0,
maxCount: p.MaxCount,
delay: p.MinDelay,
maxDelay: maxDelay,
}
}
// Do executes f with retrying policy.
// It is a shorthand of Policy.Start and Retrier.Continue.
// If f returns an error, retry to execute f until f returns nil error.
// If the error is wrapped by [MarkTemporary], Do doesn't retry and returns the error.
func (p *Policy) Do(ctx context.Context, f func() error) error {
var err error
var target *temporary
retrier := p.Start(ctx)
for retrier.Continue() {
err = f()
if err == nil {
return nil
}
// short cut for calling Unwrap
if err, ok := err.(*myError); ok {
if err.tmp {
continue
}
return err.error
}
if target == nil {
// lazy allocation of target
target = new(temporary)
}
if errors.As(err, target) {
if !(*target).temporary() {
return err
}
}
}
if err := retrier.err; err != nil {
return err
}
if err, ok := err.(*myError); ok {
// Unwrap the error if it's marked as temporary.
return err.error
}
return err
}
type temporary interface {
temporary() bool
}
var _ temporary = (*myError)(nil)
type myError struct {
error
tmp bool
}
func (e *myError) temporary() bool {
return e.tmp
}
// Unwrap implements errors.Wrapper.
func (e *myError) Unwrap() error {
return e.error
}
// MarkPermanent marks err as a permanent error, allowing retry mechanisms to handle it appropriately.
// This is especially useful in scenarios where errors require immediate termination of a process.
func MarkPermanent(err error) error {
return &myError{err, false}
}
// MarkTemporary wraps an error as a temporary error, allowing retry mechanisms to handle it appropriately.
// This is especially useful in scenarios where errors may not require immediate termination of a process,
// but rather can be resolved through retrying operations.
func MarkTemporary(err error) error {
return &myError{err, true}
}
// Continue returns whether retrying should be continued.
func (r *Retrier) Continue() bool {
r.count++
if r.count == 1 {
// always execute at first.
return true
}
if r.maxCount > 0 && r.count > r.maxCount {
// max retry count is exceeded.
return false
}
if err := r.sleepContext(r.ctx, r.delay+r.policy.randomJitter()); err != nil {
r.err = err
return false
}
// exponential back off
r.delay *= 2
if r.delay > r.maxDelay {
r.delay = r.maxDelay
}
return true
}
// Err return the error that occurred during deploy.
func (r *Retrier) Err() error {
return r.err
}
var testSleep func(ctx context.Context, d time.Duration) error
// Context supported time.Sleep
func (r *Retrier) sleepContext(ctx context.Context, d time.Duration) error {
if testSleep != nil {
return testSleep(ctx, d)
}
if d <= 0 {
return nil
}
if deadline, ok := ctx.Deadline(); ok {
if time.Until(deadline) < d {
// skip sleeping.
// because sleepContext returns context.DeadlineExceeded even if a sleep is got.
return context.DeadlineExceeded
}
}
t := r.timer
if t == nil {
t = time.NewTimer(d)
r.timer = t
} else {
t.Reset(d)
}
defer t.Stop()
select {
case <-t.C:
return nil
case <-ctx.Done():
r.timer = nil
return ctx.Err()
}
}