Add flags for client QPS and burst configuration

The recommended settings are 1:15:20 for concurrency:qps:burst. But they
should be independently configurable here. A higher-level controller
like a helm chart or addon-controller should make it simpler to adjust
these all at once in a supported way, while still allowing for full
overrides.

Refs:
 - https://issues.redhat.com/browse/ACM-4297

Signed-off-by: Justin Kulikauskas <jkulikau@redhat.com>

deploy/manager/manager.yaml

@@ -24,6 +24,8 @@ spec:
  - "--enable-lease=true"
  - "--log-level=2"
  - "--v=0"
+ - "--client-max-qps=35"
+ - "--client-burst=50"
  imagePullPolicy: Always
  env:
  - name: WATCH_NAMESPACE

deploy/operator.yaml

@@ -49,6 +49,8 @@ spec:
  - --enable-lease=true
  - --log-level=2
  - --v=0
+ - --client-max-qps=35
+ - --client-burst=50
  command:
  - config-policy-controller
  env:

main.go

@@ -73,6 +73,8 @@ type ctrlOpts struct {
  targetKubeConfig string
  metricsAddr string
  probeAddr string
+ clientQPS float32
+ clientBurst uint
  frequency uint
  decryptionConcurrency uint8
  evaluationConcurrency uint8
@@ -152,6 +154,9 @@ func main() {
  os.Exit(1)
  }
 
+ cfg.Burst = int(opts.clientBurst)
+ cfg.QPS = opts.clientQPS
+
  // Set a field selector so that a watch on CRDs will be limited to just the configuration policy CRD.
  cacheSelectors := cache.SelectorsByObject{
  &extensionsv1.CustomResourceDefinition{}: {
@@ -514,5 +519,19 @@ func setupOpts(flags *pflag.FlagSet) *ctrlOpts {
  "Disable custom metrics collection",
  )
 
+ flags.Float32Var(
+ &opts.clientQPS,
+ "client-max-qps",
+ 30, // 15 * concurrency is recommended
+ "The max queries per second that will be made against the kubernetes API server.",
+ )
+
+ flags.UintVar(
+ &opts.clientBurst,
+ "client-burst",
+ 45, // the controller-runtime defaults are 20:30 (qps:burst) - this matches that ratio
+ "The maximum burst before client requests will be throttled.",
+ )
+
  return opts
 }