diff --git a/cmd/app/options/options.go b/cmd/app/options/options.go index 0037f93f..4115305f 100644 --- a/cmd/app/options/options.go +++ b/cmd/app/options/options.go @@ -75,6 +75,11 @@ type OptionsController struct { // ConfigMapNamespaceSelector is the selector to filter on the namespaces that // receives the istio-root-ca ConfigMap ConfigMapNamespaceSelector string + + // DisableKubernetesClientRateLimiter allows the default client-go rate limiter to be disabled + // if the Kubernetes API server supports + // [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). + DisableKubernetesClientRateLimiter bool } func New() *Options { @@ -132,6 +137,14 @@ func (o *Options) Complete() error { return fmt.Errorf("failed to build kubernetes rest config: %s", err) } + if o.Controller.DisableKubernetesClientRateLimiter { + log.Info("Disabling Kubernetes client rate limiter.") + // A negative QPS and Burst indicates that the client should not have a rate limiter. + // Ref: https://github.com/kubernetes/kubernetes/blob/v1.24.0/staging/src/k8s.io/client-go/rest/config.go#L354-L364 + o.RestConfig.QPS = -1 + o.RestConfig.Burst = -1 + } + if len(o.TLS.RootCAsCertFile) == 0 { log.Info("WARNING: --root-ca-file is not defined which means the root CA will be discovered by the configured issuer. Without a statically defined trust bundle, it will be very difficult to safely rotate the chain used for issuance.") } else { @@ -287,4 +300,9 @@ func (o *Options) addControllerFlags(fs *pflag.FlagSet) { "configmap-namespace-selector", "", "Selector to filter on namespaces where the controller creates istio-ca-root-cert"+ " ConfigMap. Supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") + + fs.BoolVar(&o.Controller.DisableKubernetesClientRateLimiter, + "disable-kubernetes-client-rate-limiter", false, + "Allows the default client-go rate limiter to be disabled if the Kubernetes API server supports "+ + "[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/)") } diff --git a/deploy/charts/istio-csr/README.md b/deploy/charts/istio-csr/README.md index 3fb11069..4b2eb69e 100644 --- a/deploy/charts/istio-csr/README.md +++ b/deploy/charts/istio-csr/README.md @@ -360,6 +360,15 @@ If set, limit where istio-csr creates configmaps with root ca certificates. If u Example: maistra.io/member-of=istio-system +#### **app.controller.disableKubernetesClientRateLimiter** ~ `bool` +> Default value: +> ```yaml +> false +> ``` + +Allows you to disable the default Kubernetes client rate limiter if istio-csr is exceeding the default QPS (5) and Burst (10) limits. For example in large clusters with many Istio workloads, restarting the Pods may cause istio-csr to send bursts Kubernetes API requests that exceed the limits of the default Kubernetes client rate limiter and istio-csr will become slow to issue certificates for your workloads. Only disable client rate limiting if the Kubernetes API server supports +[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/), +to avoid overloading the server. #### **volumes** ~ `array` > Default value: > ```yaml diff --git a/deploy/charts/istio-csr/templates/deployment.yaml b/deploy/charts/istio-csr/templates/deployment.yaml index 9e2e57d9..46954f4c 100644 --- a/deploy/charts/istio-csr/templates/deployment.yaml +++ b/deploy/charts/istio-csr/templates/deployment.yaml @@ -90,6 +90,7 @@ spec: {{- if .Values.app.controller.configmapNamespaceSelector }} - "--configmap-namespace-selector={{ .Values.app.controller.configmapNamespaceSelector }}" {{- end }} + - "--disable-kubernetes-client-rate-limiter={{ .Values.app.controller.disableKubernetesClientRateLimiter }}" - "--runtime-issuance-config-map-name={{.Values.app.runtimeIssuanceConfigMap}}" - "--runtime-issuance-config-map-namespace={{.Release.Namespace}}" diff --git a/deploy/charts/istio-csr/values.schema.json b/deploy/charts/istio-csr/values.schema.json index 9589fe33..76d8703d 100644 --- a/deploy/charts/istio-csr/values.schema.json +++ b/deploy/charts/istio-csr/values.schema.json @@ -159,6 +159,9 @@ "configmapNamespaceSelector": { "$ref": "#/$defs/helm-values.app.controller.configmapNamespaceSelector" }, + "disableKubernetesClientRateLimiter": { + "$ref": "#/$defs/helm-values.app.controller.disableKubernetesClientRateLimiter" + }, "leaderElectionNamespace": { "$ref": "#/$defs/helm-values.app.controller.leaderElectionNamespace" } @@ -169,6 +172,11 @@ "description": "If set, limit where istio-csr creates configmaps with root ca certificates. If unset, configmap created in ALL namespaces.\nExample: maistra.io/member-of=istio-system", "type": "string" }, + "helm-values.app.controller.disableKubernetesClientRateLimiter": { + "default": false, + "description": "Allows you to disable the default Kubernetes client rate limiter if istio-csr is exceeding the default QPS (5) and Burst (10) limits. For example in large clusters with many Istio workloads, restarting the Pods may cause istio-csr to send bursts Kubernetes API requests that exceed the limits of the default Kubernetes client rate limiter and istio-csr will become slow to issue certificates for your workloads. Only disable client rate limiting if the Kubernetes API server supports\n[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/),\nto avoid overloading the server.", + "type": "boolean" + }, "helm-values.app.controller.leaderElectionNamespace": { "default": "istio-system", "type": "string" diff --git a/deploy/charts/istio-csr/values.yaml b/deploy/charts/istio-csr/values.yaml index 945e8cb2..34aea9d4 100644 --- a/deploy/charts/istio-csr/values.yaml +++ b/deploy/charts/istio-csr/values.yaml @@ -195,6 +195,17 @@ app: # +docs:property # configmapNamespaceSelector: + # Allows you to disable the default Kubernetes client rate limiter if + # istio-csr is exceeding the default QPS (5) and Burst (10) limits. + # For example in large clusters with many Istio workloads, restarting the Pods may cause + # istio-csr to send bursts Kubernetes API requests that exceed the limits of + # the default Kubernetes client rate limiter and istio-csr will become slow to issue + # certificates for your workloads. + # Only disable client rate limiting if the Kubernetes API server supports + # [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/), + # to avoid overloading the server. + disableKubernetesClientRateLimiter: false + # Optional extra volumes. Useful for mounting custom root CAs # # For example: diff --git a/make/test-e2e.mk b/make/test-e2e.mk index a1c5d59a..4536b138 100644 --- a/make/test-e2e.mk +++ b/make/test-e2e.mk @@ -68,6 +68,7 @@ test-e2e-deps: INSTALL_OPTIONS := test-e2e-deps: INSTALL_OPTIONS += --set image.repository=$(oci_manager_image_name_development) test-e2e-deps: INSTALL_OPTIONS += --set app.runtimeIssuanceConfigMap=$(E2E_RUNTIME_CONFIG_MAP_NAME) test-e2e-deps: INSTALL_OPTIONS += --set app.logFormat=json +test-e2e-deps: INSTALL_OPTIONS += --set app.controller.disableKubernetesClientRateLimiter=true test-e2e-deps: INSTALL_OPTIONS += -f ./make/config/istio-csr-values.yaml test-e2e-deps: e2e-setup-cert-manager test-e2e-deps: e2e-create-cert-manager-istio-resources