Merge pull request #788 from hashicorp/lkysow/crd-backoff

Change CRD retry-backoff

CHANGELOG.md

@@ -10,15 +10,15 @@ FEATURES:
 IMPROVEMENTS:
 * Control Plane
   * Upgrade Docker image Alpine version from 3.13 to 3.14. [[GH-737](https://github.com/hashicorp/consul-k8s/pull/737)]
+  * CRDs: tune failure backoff so invalid config entries are re-synced more quickly. [[GH-788](https://github.com/hashicorp/consul-k8s/pull/788)]
 * Helm Chart
   * Enable adding extra containers to server and client Pods. [[GH-749](https://github.com/hashicorp/consul-k8s/pull/749)]
   * ACL support for Admin Partitions. **(Consul Enterprise only)**
   **BETA** [[GH-766](https://github.com/hashicorp/consul-k8s/pull/766)]
     * This feature now enabled ACL support for Admin Partitions. The server-acl-init job now creates a Partition token. This token
-can be used to bootstrap new partitions as well as manage ACLs in the non-default partitions.
+      can be used to bootstrap new partitions as well as manage ACLs in the non-default partitions.
     * Partition to partition networking is disabled if ACLs are enabled.
     * Documentation for the installation can be found [here](https://github.com/hashicorp/consul-k8s/blob/main/docs/admin-partitions-with-acls.md).
-
 * CLI
   * Add `version` command. [[GH-741](https://github.com/hashicorp/consul-k8s/pull/741)]
   * Add `uninstall` command. [[GH-725](https://github.com/hashicorp/consul-k8s/pull/725)]

control-plane/controller/configentry_controller.go

@@ -11,12 +11,16 @@ import (
 	"github.com/hashicorp/consul-k8s/control-plane/api/common"
 	"github.com/hashicorp/consul-k8s/control-plane/namespaces"
 	capi "github.com/hashicorp/consul/api"
+	"golang.org/x/time/rate"
 	corev1 "k8s.io/api/core/v1"
 	k8serr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 const (
@@ -246,6 +250,37 @@ func (r *ConfigEntryController) ReconcileEntry(ctx context.Context, crdCtrl Cont
 	return ctrl.Result{}, nil
 }
 
+// setupWithManager sets up the controller manager for the given resource
+// with our default options.
+func setupWithManager(mgr ctrl.Manager, resource client.Object, reconciler reconcile.Reconciler) error {
+	options := controller.Options{
+		// Taken from https://github.com/kubernetes/client-go/blob/master/util/workqueue/default_rate_limiters.go#L39
+		// and modified from a starting backoff of 5ms and max of 1000s to a
+		// starting backoff of 200ms and a max of 5s to better fit our most
+		// common error cases and performance characteristics.
+		//
+		// One common error case is that a config entry is applied that requires
+		// a protocol like http or grpc. Often the user will apply a new config
+		// entry to set the protocol in a minute or two. During this time, the
+		// default backoff could then be set up to 5m or more which means the
+		// original config entry takes a long time to re-sync.
+		//
+		// In terms of performance, Consul servers can handle tens of thousands
+		// of writes per second, so retrying at max every 5s isn't an issue and
+		// provides a better UX.
+		RateLimiter: workqueue.NewMaxOfRateLimiter(
+			workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 5*time.Second),
+			// 10 qps, 100 bucket size.  This is only for retry speed and its only the overall factor (not per item)
+			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+		),
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(resource).
+		WithOptions(options).
+		Complete(reconciler)
+}
+
 func (r *ConfigEntryController) consulNamespace(configEntry capi.ConfigEntry, namespace string, globalResource bool) string {
 	// ServiceIntentions have the appropriate Consul Namespace set on them as the value
 	// is defaulted by the webhook. These are then set on the ServiceIntentions config entry

control-plane/controller/ingressgateway_controller.go

@@ -36,7 +36,5 @@ func (r *IngressGatewayController) UpdateStatus(ctx context.Context, obj client.
 }
 
 func (r *IngressGatewayController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.IngressGateway{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.IngressGateway{}, r)
 }

control-plane/controller/mesh_controller.go

@@ -36,7 +36,5 @@ func (r *MeshController) UpdateStatus(ctx context.Context, obj client.Object, op
 }
 
 func (r *MeshController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.Mesh{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.Mesh{}, r)
 }

control-plane/controller/proxydefaults_controller.go

@@ -36,7 +36,5 @@ func (r *ProxyDefaultsController) UpdateStatus(ctx context.Context, obj client.O
 }
 
 func (r *ProxyDefaultsController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.ProxyDefaults{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.ProxyDefaults{}, r)
 }

control-plane/controller/servicedefaults_controller.go

@@ -36,7 +36,5 @@ func (r *ServiceDefaultsController) UpdateStatus(ctx context.Context, obj client
 }
 
 func (r *ServiceDefaultsController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.ServiceDefaults{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.ServiceDefaults{}, r)
 }

control-plane/controller/serviceintentions_controller.go

@@ -36,7 +36,5 @@ func (r *ServiceIntentionsController) UpdateStatus(ctx context.Context, obj clie
 }
 
 func (r *ServiceIntentionsController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.ServiceIntentions{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.ServiceIntentions{}, r)
 }

control-plane/controller/serviceresolver_controller.go

@@ -36,7 +36,5 @@ func (r *ServiceResolverController) UpdateStatus(ctx context.Context, obj client
 }
 
 func (r *ServiceResolverController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.ServiceResolver{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.ServiceResolver{}, r)
 }

control-plane/controller/servicerouter_controller.go

@@ -36,7 +36,5 @@ func (r *ServiceRouterController) UpdateStatus(ctx context.Context, obj client.O
 }
 
 func (r *ServiceRouterController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.ServiceRouter{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.ServiceRouter{}, r)
 }

control-plane/controller/servicesplitter_controller.go

@@ -36,7 +36,5 @@ func (r *ServiceSplitterController) UpdateStatus(ctx context.Context, obj client
 }
 
 func (r *ServiceSplitterController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.ServiceSplitter{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.ServiceSplitter{}, r)
 }

control-plane/controller/terminatinggateway_controller.go

@@ -36,7 +36,5 @@ func (r *TerminatingGatewayController) UpdateStatus(ctx context.Context, obj cli
 }
 
 func (r *TerminatingGatewayController) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&consulv1alpha1.TerminatingGateway{}).
-		Complete(r)
+	return setupWithManager(mgr, &consulv1alpha1.TerminatingGateway{}, r)
 }

control-plane/go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	go.uber.org/zap v1.17.0
 	golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	golang.org/x/tools v0.1.2 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0
 	k8s.io/api v0.21.1