Add flags for client QPS and burst configuration

The recommended settings are 2:30:45 for concurrency:qps:burst. When QPS
or Burst are not explicitly set, they will scale based on concurrency.

Refs:
 - https://issues.redhat.com/browse/ACM-4297

Signed-off-by: Justin Kulikauskas <jkulikau@redhat.com>

deploy/manager/manager.yaml

@@ -24,6 +24,8 @@ spec:
             - "--enable-lease=true"
             - "--log-level=2"
             - "--v=0"
+            - "--client-max-qps=35"
+            - "--client-burst=50"
           imagePullPolicy: Always
           env:
             - name: WATCH_NAMESPACE

deploy/operator.yaml

@@ -49,6 +49,8 @@ spec:
         - --enable-lease=true
         - --log-level=2
         - --v=0
+        - --client-max-qps=35
+        - --client-burst=50
         command:
         - config-policy-controller
         env:

main.go

@@ -73,6 +73,8 @@ type ctrlOpts struct {
 	targetKubeConfig      string
 	metricsAddr           string
 	probeAddr             string
+	clientQPS             float32
+	clientBurst           uint
 	frequency             uint
 	decryptionConcurrency uint8
 	evaluationConcurrency uint8
@@ -112,9 +114,7 @@ func main() {
 	zflags.Bind(flag.CommandLine)
 	controllerFlagSet.AddGoFlagSet(flag.CommandLine)
 
-	opts := setupOpts(controllerFlagSet)
-
-	_ = controllerFlagSet.Parse(os.Args[2:])
+	opts := parseOpts(controllerFlagSet, os.Args[2:])
 
 	ctrlZap, err := zflags.BuildForCtrl()
 	if err != nil {
@@ -151,6 +151,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	cfg.Burst = int(opts.clientBurst)
+	cfg.QPS = opts.clientQPS
+
 	// Set a field selector so that a watch on CRDs will be limited to just the configuration policy CRD.
 	cacheSelectors := cache.SelectorsByObject{
 		&extensionsv1.CustomResourceDefinition{}: {
@@ -424,7 +427,7 @@ func handleTriggerUninstall() {
 	}
 }
 
-func setupOpts(flags *pflag.FlagSet) *ctrlOpts {
+func parseOpts(flags *pflag.FlagSet, args []string) *ctrlOpts {
 	opts := &ctrlOpts{}
 
 	flags.UintVar(
@@ -513,5 +516,34 @@ func setupOpts(flags *pflag.FlagSet) *ctrlOpts {
 		"Disable custom metrics collection",
 	)
 
+	flags.Float32Var(
+		&opts.clientQPS,
+		"client-max-qps",
+		30, // 15 * concurrency is recommended
+		"The max queries per second that will be made against the kubernetes API server. "+
+			"Will scale with concurrency, if not explicitly set.",
+	)
+
+	flags.UintVar(
+		&opts.clientBurst,
+		"client-burst",
+		45, // the controller-runtime defaults are 20:30 (qps:burst) - this matches that ratio
+		"The maximum burst before client requests will be throttled. "+
+			"Will scale with concurrency, if not explicitly set.",
+	)
+
+	_ = flags.Parse(args)
+
+	// Scale QPS and Burst with concurrency, when they aren't explicitly set.
+	if flags.Changed("evaluation-concurrency") {
+		if !flags.Changed("client-max-qps") {
+			opts.clientQPS = float32(opts.evaluationConcurrency) * 15
+		}
+
+		if !flags.Changed("client-burst") {
+			opts.clientBurst = uint(opts.evaluationConcurrency)*22 + 1
+		}
+	}
+
 	return opts
 }