From 9d48b6299d0d36626989ae5671b2d3e0e52b6f44 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Fri, 26 Feb 2021 12:14:22 +0200 Subject: [PATCH] Retry with exponential backoff when fetching artifacts Signed-off-by: Stefan Prodan --- controllers/kustomization_controller.go | 39 +++++++++++++++---------- go.mod | 1 + main.go | 3 ++ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index e57ec50a..fe007d19 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -36,6 +37,7 @@ import ( "github.com/fluxcd/pkg/untar" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/go-logr/logr" + "github.com/hashicorp/go-retryablehttp" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -68,6 +70,7 @@ import ( // KustomizationReconciler reconciles a Kustomization object type KustomizationReconciler struct { client.Client + httpClient *retryablehttp.Client requeueDependency time.Duration Scheme *runtime.Scheme EventRecorder kuberecorder.EventRecorder @@ -78,6 +81,7 @@ type KustomizationReconciler struct { type KustomizationReconcilerOptions struct { MaxConcurrentReconciles int + HTTPRetry int DependencyRequeueInterval time.Duration } @@ -96,6 +100,15 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom r.requeueDependency = opts.DependencyRequeueInterval + // Configure the retryable http client used for fetching artifacts. + // By default it retries 10 times within a 3.5 minutes window. + httpClient := retryablehttp.NewClient() + httpClient.RetryWaitMin = 5 * time.Second + httpClient.RetryWaitMax = 30 * time.Second + httpClient.RetryMax = opts.HTTPRetry + httpClient.Logger = nil + r.httpClient = httpClient + return ctrl.NewControllerManagedBy(mgr). For(&kustomizev1.Kustomization{}, builder.WithPredicates( predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}), @@ -410,38 +423,34 @@ func (r *KustomizationReconciler) checkDependencies(kustomization kustomizev1.Ku return nil } -func (r *KustomizationReconciler) download(kustomization kustomizev1.Kustomization, url string, tmpDir string) error { +func (r *KustomizationReconciler) download(kustomization kustomizev1.Kustomization, artifactURL string, tmpDir string) error { timeout := kustomization.GetTimeout() + (time.Second * 1) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" { - namespace := kustomization.GetNamespace() - if kustomization.Spec.SourceRef.Namespace != "" { - namespace = kustomization.Spec.SourceRef.Namespace + u, err := url.Parse(artifactURL) + if err != nil { + return err } - url = fmt.Sprintf("http://%s/%s/%s/%s/latest.tar.gz", - hostname, - strings.ToLower(kustomization.Spec.SourceRef.Kind), - namespace, - kustomization.Spec.SourceRef.Name) + u.Host = hostname + artifactURL = u.String() } - // download the tarball - req, err := http.NewRequest("GET", url, nil) + req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil) if err != nil { - return fmt.Errorf("failed to create HTTP request for %s, error: %w", url, err) + return fmt.Errorf("failed to create a new request: %w", err) } - resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + resp, err := r.httpClient.Do(req.WithContext(ctx)) if err != nil { - return fmt.Errorf("failed to download artifact from %s, error: %w", url, err) + return fmt.Errorf("failed to download artifact, error: %w", err) } defer resp.Body.Close() // check response if resp.StatusCode != http.StatusOK { - return fmt.Errorf("faild to download artifact from %s, status: %s", url, resp.Status) + return fmt.Errorf("faild to download artifact from %s, status: %s", artifactURL, resp.Status) } // extract diff --git a/go.mod b/go.mod index 26bc15de..bed7f2ce 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/fluxcd/pkg/untar v0.0.5 github.com/fluxcd/source-controller/api v0.9.0 github.com/go-logr/logr v0.3.0 + github.com/hashicorp/go-retryablehttp v0.6.8 github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c github.com/onsi/ginkgo v1.14.2 github.com/onsi/gomega v1.10.2 diff --git a/main.go b/main.go index f1c17ca7..c7309b35 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ func main() { clientOptions client.Options logOptions logger.Options watchAllNamespaces bool + httpRetry int ) flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -77,6 +78,7 @@ func main() { flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.") flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") + flag.IntVar(&httpRetry, "http-retry", 9, "The maximum number of retries when failing to fetch artifacts over HTTP.") flag.Bool("log-json", false, "Set logging to JSON format.") flag.CommandLine.MarkDeprecated("log-json", "Please use --log-encoding=json instead.") clientOptions.BindFlags(flag.CommandLine) @@ -132,6 +134,7 @@ func main() { }).SetupWithManager(mgr, controllers.KustomizationReconcilerOptions{ MaxConcurrentReconciles: concurrent, DependencyRequeueInterval: requeueDependency, + HTTPRetry: httpRetry, }); err != nil { setupLog.Error(err, "unable to create controller", "controller", kustomizev1.KustomizationKind) os.Exit(1)