From 2700bed1baa05d413d4743fcc5e8c36907750066 Mon Sep 17 00:00:00 2001 From: Kevin Nisbet Date: Tue, 28 Sep 2021 10:05:25 -0400 Subject: [PATCH] add temporary failures as a reason to retry kubernetes failures (#2651) (cherry picked from commit 03d0169cee6d3f3f5b19d96a40e984dafcd973e3) --- lib/kubernetes/errors.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/kubernetes/errors.go b/lib/kubernetes/errors.go index fe1e8ceba0..aa5f46de37 100644 --- a/lib/kubernetes/errors.go +++ b/lib/kubernetes/errors.go @@ -18,19 +18,26 @@ package kubernetes import ( "github.com/cenkalti/backoff" + "github.com/gravitational/gravity/lib/utils" "github.com/gravitational/rigging" "github.com/gravitational/trace" "k8s.io/apimachinery/pkg/api/errors" ) -// RetryOnUpdateConflict retries on update conflict errors +// RetryOnUpdateConflict retries on update conflict errors (and temporary failures). func RetryOnUpdateConflict(err error) error { if err == nil { return nil } origErr := trace.Unwrap(err) switch { - case errors.IsConflict(origErr): + case errors.IsConflict(origErr), + errors.IsServiceUnavailable(origErr), + errors.IsServerTimeout(origErr), + errors.IsTimeout(origErr), + errors.IsInternalError(origErr), + errors.IsTooManyRequests(origErr), + utils.IsTransientClusterError(origErr): return rigging.ConvertError(origErr) default: return &backoff.PermanentError{Err: err}