feat: Support specifying the pattern for transient and retryable erro…

…rs (#4889)

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

util/errors/errors.go

@@ -3,6 +3,8 @@ package errors
 import (
 	"net"
 	"net/url"
+	"os"
+	"regexp"
 	"strings"
 
 	apierr "k8s.io/apimachinery/pkg/api/errors"
@@ -15,7 +17,18 @@ func IsTransientErr(err error) bool {
 		return false
 	}
 	err = argoerrs.Cause(err)
-	return isExceededQuotaErr(err) || apierr.IsTooManyRequests(err) || isResourceQuotaConflictErr(err) || isTransientNetworkErr(err) || apierr.IsServerTimeout(err) || apierr.IsServiceUnavailable(err)
+	return isExceededQuotaErr(err) || apierr.IsTooManyRequests(err) || isResourceQuotaConflictErr(err) || isTransientNetworkErr(err) || apierr.IsServerTimeout(err) || apierr.IsServiceUnavailable(err) || matchTransientErrPattern(err)
+}
+
+func matchTransientErrPattern(err error) bool {
+	// TRANSIENT_ERROR_PATTERN allows to specify the pattern to match for errors that can be seen as transient
+	// and retryable.
+	pattern, _ := os.LookupEnv("TRANSIENT_ERROR_PATTERN")
+	if pattern == "" {
+		return false
+	}
+	match, _ := regexp.MatchString(pattern, err.Error())
+	return match
 }
 
 func isExceededQuotaErr(err error) bool {

util/errors/errors_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"net"
 	"net/url"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -20,6 +21,9 @@ func (n netError) Temporary() bool { return false }
 var tlsHandshakeTimeoutErr net.Error = netError("net/http: TLS handshake timeout")
 var ioTimeoutErr net.Error = netError("i/o timeout")
 var connectionTimedout net.Error = netError("connection timed out")
+var transientErr net.Error = netError("this error is transient")
+
+const transientEnvVarKey = "TRANSIENT_ERROR_PATTERN"
 
 func TestIsTransientErr(t *testing.T) {
 	t.Run("Nil", func(t *testing.T) {
@@ -58,4 +62,16 @@ func TestIsTransientErr(t *testing.T) {
 	t.Run("ConnectionTimeout", func(t *testing.T) {
 		assert.True(t, IsTransientErr(connectionTimedout))
 	})
+	t.Run("TransientErrorPattern", func(t *testing.T) {
+		_ = os.Setenv(transientEnvVarKey, "this error is transient")
+		assert.True(t, IsTransientErr(transientErr))
+
+		_ = os.Setenv(transientEnvVarKey, "this error is not transient")
+		assert.False(t, IsTransientErr(transientErr))
+
+		_ = os.Setenv(transientEnvVarKey, "")
+		assert.False(t, IsTransientErr(transientErr))
+
+		_ = os.Unsetenv(transientEnvVarKey)
+	})
 }