-
Notifications
You must be signed in to change notification settings - Fork 3.2k
/
errors.go
146 lines (129 loc) · 4.14 KB
/
errors.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
package errors
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"os/exec"
"regexp"
"strings"
log "github.com/sirupsen/logrus"
apierr "k8s.io/apimachinery/pkg/api/errors"
argoerrs "github.com/argoproj/argo-workflows/v3/errors"
)
func IgnoreContainerNotFoundErr(err error) error {
if err != nil && strings.Contains(err.Error(), "container not found") {
return nil
}
return err
}
// IsTransientErr reports whether the error is transient and logs it.
func IsTransientErr(err error) bool {
isTransient := IsTransientErrQuiet(err)
if !isTransient {
log.Warnf("Non-transient error: %v", err)
}
return isTransient
}
// IsTransientErrQuiet reports whether the error is transient and logs only if it is.
func IsTransientErrQuiet(err error) bool {
isTransient := isTransientErr(err)
if isTransient {
log.Infof("Transient error: %v", err)
}
return isTransient
}
// isTransientErr reports whether the error is transient.
func isTransientErr(err error) bool {
if err == nil {
return false
}
err = argoerrs.Cause(err)
return isExceededQuotaErr(err) ||
apierr.IsTooManyRequests(err) ||
isResourceQuotaConflictErr(err) ||
isResourceQuotaTimeoutErr(err) ||
isTransientNetworkErr(err) ||
apierr.IsServerTimeout(err) ||
apierr.IsServiceUnavailable(err) ||
isTransientEtcdErr(err) ||
matchTransientErrPattern(err) ||
errors.Is(err, NewErrTransient("")) ||
isTransientSqbErr(err)
}
func matchTransientErrPattern(err error) bool {
// TRANSIENT_ERROR_PATTERN allows to specify the pattern to match for errors that can be seen as transient
// and retryable.
pattern, _ := os.LookupEnv("TRANSIENT_ERROR_PATTERN")
if pattern == "" {
return false
}
match, _ := regexp.MatchString(pattern, generateErrorString(err))
return match
}
func isExceededQuotaErr(err error) bool {
return apierr.IsForbidden(err) && strings.Contains(err.Error(), "exceeded quota")
}
func isResourceQuotaConflictErr(err error) bool {
return apierr.IsConflict(err) && strings.Contains(err.Error(), "Operation cannot be fulfilled on resourcequota")
}
func isResourceQuotaTimeoutErr(err error) bool {
return apierr.IsInternalError(err) && strings.Contains(err.Error(), "resource quota evaluation timed out")
}
func isTransientEtcdErr(err error) bool {
// Some clusters expose these (transient) etcd errors to the caller
if strings.Contains(err.Error(), "etcdserver: leader changed") {
return true
} else if strings.Contains(err.Error(), "etcdserver: request timed out") {
return true
}
return false
}
func isTransientNetworkErr(err error) bool {
switch err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
return true
}
errorString := generateErrorString(err)
if strings.Contains(errorString, "Connection closed by foreign host") {
// For a URL error, where it replies back "connection closed"
// retry again.
return true
} else if strings.Contains(errorString, "net/http: TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError, retry.
return true
} else if strings.Contains(errorString, "i/o timeout") {
// If error is - tcp timeoutError, retry.
return true
} else if strings.Contains(errorString, "connection timed out") {
// If err is a net.Dial timeout, retry.
return true
} else if strings.Contains(errorString, "connection reset by peer") {
// If err is a ECONNRESET, retry.
return true
} else if _, ok := err.(*url.Error); ok && strings.Contains(errorString, "EOF") {
// If err is EOF, retry.
return true
} else if strings.Contains(errorString, "http2: client connection lost") {
// If err is http2 transport ping timeout, retry.
return true
} else if strings.Contains(errorString, "connect: connection refused") {
// If err is connection refused, retry.
return true
} else if strings.Contains(errorString, "invalid connection") {
// If err is invalid connection, retry.
return true
}
return false
}
func generateErrorString(err error) string {
errorString := err.Error()
if exitErr, ok := err.(*exec.ExitError); ok {
errorString = fmt.Sprintf("%s %s", errorString, exitErr.Stderr)
}
return errorString
}
func isTransientSqbErr(err error) bool {
return strings.Contains(err.Error(), "upper: no more rows in")
}