Merge pull request #352 from seankhliao/unlimited-qps

Disable kubernetes client side rate limiting

cmd/app/options/options.go

@@ -75,6 +75,11 @@ type OptionsController struct {
 	// ConfigMapNamespaceSelector is the selector to filter on the namespaces that
 	// receives the istio-root-ca ConfigMap
 	ConfigMapNamespaceSelector string
+
+	// DisableKubernetesClientRateLimiter allows the default client-go rate limiter to be disabled
+	// if the Kubernetes API server supports
+	// [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/).
+	DisableKubernetesClientRateLimiter bool
 }
 
 func New() *Options {
@@ -132,6 +137,14 @@ func (o *Options) Complete() error {
 		return fmt.Errorf("failed to build kubernetes rest config: %s", err)
 	}
 
+	if o.Controller.DisableKubernetesClientRateLimiter {
+		log.Info("Disabling Kubernetes client rate limiter.")
+		// A negative QPS and Burst indicates that the client should not have a rate limiter.
+		// Ref: https://github.com/kubernetes/kubernetes/blob/v1.24.0/staging/src/k8s.io/client-go/rest/config.go#L354-L364
+		o.RestConfig.QPS = -1
+		o.RestConfig.Burst = -1
+	}
+
 	if len(o.TLS.RootCAsCertFile) == 0 {
 		log.Info("WARNING: --root-ca-file is not defined which means the root CA will be discovered by the configured issuer. Without a statically defined trust bundle, it will be very difficult to safely rotate the chain used for issuance.")
 	} else {
@@ -287,4 +300,9 @@ func (o *Options) addControllerFlags(fs *pflag.FlagSet) {
 		"configmap-namespace-selector", "",
 		"Selector to filter on namespaces where the controller creates istio-ca-root-cert"+
 			" ConfigMap. Supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)")
+
+	fs.BoolVar(&o.Controller.DisableKubernetesClientRateLimiter,
+		"disable-kubernetes-client-rate-limiter", false,
+		"Allows the default client-go rate limiter to be disabled if the Kubernetes API server supports "+
+			"[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/)")
 }

deploy/charts/istio-csr/README.md

@@ -360,6 +360,15 @@ If set, limit where istio-csr creates configmaps with root ca certificates. If u
 Example: maistra.io/member-of=istio-system
 
 
+#### **app.controller.disableKubernetesClientRateLimiter** ~ `bool`
+> Default value:
+> ```yaml
+> false
+> ```
+
+Allows you to disable the default Kubernetes client rate limiter if istio-csr is exceeding the default QPS (5) and Burst (10) limits. For example in large clusters with many Istio workloads, restarting the Pods may cause istio-csr to send bursts Kubernetes API requests that exceed the limits of the default Kubernetes client rate limiter and istio-csr will become slow to issue certificates for your workloads. Only disable client rate limiting if the Kubernetes API server supports  
+[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/),  
+to avoid overloading the server.
 #### **volumes** ~ `array`
 > Default value:
 > ```yaml

deploy/charts/istio-csr/templates/deployment.yaml

@@ -90,6 +90,7 @@ spec:
           {{- if .Values.app.controller.configmapNamespaceSelector }}
           - "--configmap-namespace-selector={{ .Values.app.controller.configmapNamespaceSelector }}"
           {{- end }}
+          - "--disable-kubernetes-client-rate-limiter={{ .Values.app.controller.disableKubernetesClientRateLimiter }}"
 
           - "--runtime-issuance-config-map-name={{.Values.app.runtimeIssuanceConfigMap}}"
           - "--runtime-issuance-config-map-namespace={{.Release.Namespace}}"

deploy/charts/istio-csr/values.schema.json

@@ -159,6 +159,9 @@
         "configmapNamespaceSelector": {
           "$ref": "#/$defs/helm-values.app.controller.configmapNamespaceSelector"
         },
+        "disableKubernetesClientRateLimiter": {
+          "$ref": "#/$defs/helm-values.app.controller.disableKubernetesClientRateLimiter"
+        },
         "leaderElectionNamespace": {
           "$ref": "#/$defs/helm-values.app.controller.leaderElectionNamespace"
         }
@@ -169,6 +172,11 @@
       "description": "If set, limit where istio-csr creates configmaps with root ca certificates. If unset, configmap created in ALL namespaces.\nExample: maistra.io/member-of=istio-system",
       "type": "string"
     },
+    "helm-values.app.controller.disableKubernetesClientRateLimiter": {
+      "default": false,
+      "description": "Allows you to disable the default Kubernetes client rate limiter if istio-csr is exceeding the default QPS (5) and Burst (10) limits. For example in large clusters with many Istio workloads, restarting the Pods may cause istio-csr to send bursts Kubernetes API requests that exceed the limits of the default Kubernetes client rate limiter and istio-csr will become slow to issue certificates for your workloads. Only disable client rate limiting if the Kubernetes API server supports\n[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/),\nto avoid overloading the server.",
+      "type": "boolean"
+    },
     "helm-values.app.controller.leaderElectionNamespace": {
       "default": "istio-system",
       "type": "string"

deploy/charts/istio-csr/values.yaml

@@ -195,6 +195,17 @@ app:
     # +docs:property
     # configmapNamespaceSelector:
 
+    # Allows you to disable the default Kubernetes client rate limiter if
+    # istio-csr is exceeding the default QPS (5) and Burst (10) limits.
+    # For example in large clusters with many Istio workloads, restarting the Pods may cause
+    # istio-csr to send bursts Kubernetes API requests that exceed the limits of
+    # the default Kubernetes client rate limiter and istio-csr will become slow to issue
+    # certificates for your workloads.
+    # Only disable client rate limiting if the Kubernetes API server supports
+    # [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/),
+    # to avoid overloading the server.
+    disableKubernetesClientRateLimiter: false
+
 # Optional extra volumes. Useful for mounting custom root CAs
 #
 # For example:

make/test-e2e.mk

@@ -68,6 +68,7 @@ test-e2e-deps: INSTALL_OPTIONS :=
 test-e2e-deps: INSTALL_OPTIONS += --set image.repository=$(oci_manager_image_name_development)
 test-e2e-deps: INSTALL_OPTIONS += --set app.runtimeIssuanceConfigMap=$(E2E_RUNTIME_CONFIG_MAP_NAME)
 test-e2e-deps: INSTALL_OPTIONS += --set app.logFormat=json
+test-e2e-deps: INSTALL_OPTIONS += --set app.controller.disableKubernetesClientRateLimiter=true
 test-e2e-deps: INSTALL_OPTIONS += -f ./make/config/istio-csr-values.yaml
 test-e2e-deps: e2e-setup-cert-manager
 test-e2e-deps: e2e-create-cert-manager-istio-resources