discovery: make gimbal client request rate limit configurable

discovery/cmd/kubernetes-discoverer/main.go

@@ -42,6 +42,8 @@ var (
 	debug                 bool
 	prometheusListenPort  int
 	discovererMetrics     localmetrics.DiscovererMetrics
+	gimbalKubeClientQPS   float64
+	gimbalKubeClientBurst int
 )
 
 func init() {
@@ -53,6 +55,8 @@ func init() {
 	flag.DurationVar(&resyncInterval, "resync-interval", time.Minute*30, "Default resync period for watcher to refresh")
 	flag.BoolVar(&debug, "debug", false, "Enable debug logging.")
 	flag.IntVar(&prometheusListenPort, "prometheus-listen-address", 8080, "The address to listen on for Prometheus HTTP requests")
+	flag.Float64Var(&gimbalKubeClientQPS, "gimbal-client-qps", 5, "The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server")
+	flag.IntVar(&gimbalKubeClientBurst, "gimbal-client-burst", 10, "The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst")
 	flag.Parse()
 }
 
@@ -69,6 +73,12 @@ func main() {
 	}
 
 	log.Info("Gimbal Kubernetes Discoverer Starting up...")
+	log.Infof("Version: %s", buildinfo.Version)
+	log.Infof("Backend name: %s", backendName)
+	log.Infof("Number of queue worker threads: %d", numProcessThreads)
+	log.Infof("Resync interval: %v", resyncInterval)
+	log.Infof("Gimbal kubernetes client QPS: %v", gimbalKubeClientQPS)
+	log.Infof("Gimbal kubernetes client burst: %d", gimbalKubeClientBurst)
 
 	// Init prometheus metrics
 	discovererMetrics = localmetrics.NewMetrics()
@@ -90,7 +100,7 @@ func main() {
 	}
 
 	// Init
-	gimbalKubeClient, err := k8s.NewClient(gimbalKubeCfgFile, log)
+	gimbalKubeClient, err := k8s.NewClientWithQPS(gimbalKubeCfgFile, log, float32(gimbalKubeClientQPS), gimbalKubeClientBurst)
 	if err != nil {
 		log.Fatal("Could not init k8sclient! ", err)
 	}

discovery/cmd/openstack-discoverer/main.go

@@ -50,6 +50,8 @@ var (
 	prometheusListenPort              int
 	discovererMetrics                 localmetrics.DiscovererMetrics
 	log                               *logrus.Logger
+	gimbalKubeClientQPS               float64
+	gimbalKubeClientBurst             int
 )
 
 const (
@@ -67,6 +69,8 @@ func init() {
 	flag.DurationVar(&httpClientTimeout, "http-client-timeout", 5*time.Second, "The HTTP client request timeout.")
 	flag.StringVar(&openstackCertificateAuthorityFile, "openstack-certificate-authority", "", "Path to cert file of the OpenStack API certificate authority.")
 	flag.IntVar(&prometheusListenPort, "prometheus-listen-address", 8080, "The address to listen on for Prometheus HTTP requests")
+	flag.Float64Var(&gimbalKubeClientQPS, "gimbal-client-qps", 5, "The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server")
+	flag.IntVar(&gimbalKubeClientBurst, "gimbal-client-burst", 10, "The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst")
 	flag.Parse()
 }
 
@@ -86,6 +90,12 @@ func main() {
 	}
 
 	log.Info("Gimbal OpenStack Discoverer Starting up...")
+	log.Infof("Version: %s", buildinfo.Version)
+	log.Infof("Backend name: %s", backendName)
+	log.Infof("Number of queue worker threads: %d", numProcessThreads)
+	log.Infof("Reconciliation period: %v", reconciliationPeriod)
+	log.Infof("Gimbal kubernetes client QPS: %v", gimbalKubeClientQPS)
+	log.Infof("Gimbal kubernetes client burst: %d", gimbalKubeClientBurst)
 
 	// Init prometheus metrics
 	discovererMetrics = localmetrics.NewMetrics()
@@ -97,7 +107,7 @@ func main() {
 	}
 	log.Infof("BackendName is: %s", backendName)
 
-	gimbalKubeClient, err := k8s.NewClient(gimbalKubeCfgFile, log)
+	gimbalKubeClient, err := k8s.NewClientWithQPS(gimbalKubeCfgFile, log, float32(gimbalKubeClientQPS), gimbalKubeClientBurst)
 	if err != nil {
 		log.Fatal("Failed to create kubernetes client", err)
 	}

discovery/pkg/k8s/client.go

@@ -31,6 +31,19 @@ func NewClient(kubeCfgFile string, logger *logrus.Logger) (kubernetes.Interface,
 	return kubernetes.NewForConfig(config)
 }
 
+// NewClientWithQPS returns a Kubernetes client using the given configuration
+// and rate limiting parameters. If no config is provided, assumes it is running
+// inside a Kubernetes cluster and uses the in-cluster config.
+func NewClientWithQPS(kubeCfgFile string, logger *logrus.Logger, qps float32, burst int) (kubernetes.Interface, error) {
+	config, err := buildConfig(kubeCfgFile, logger)
+	if err != nil {
+		return nil, err
+	}
+	config.QPS = qps
+	config.Burst = burst
+	return kubernetes.NewForConfig(config)
+}
+
 func buildConfig(kubeCfgFile string, logger *logrus.Logger) (*rest.Config, error) {
 	if kubeCfgFile != "" {
 		logger.Infof("Using OutOfCluster k8s config with kubeConfigFile: %s", kubeCfgFile)

docs/README.md

@@ -1,6 +1,6 @@
 # Documentation
 
-See the root-level README for an introduction, and the [deployment directory](../deployment/README.md) to get started with setting up and and deploying Gimbal.
+See the root-level README for an introduction, and the [deployment directory](../deployment/README.md) to get started with setting up and deploying Gimbal.
 
 Here you can dig into the details of how Gimbal works, and explore more advanced topics for operators and users.
 

docs/kubernetes-discoverer.md

@@ -4,7 +4,7 @@
 
 The Kubernetes discoverer provides service discovery for a Kubernetes cluster. It does this by monitoring available Services and Endpoints for a single Kubernetes cluster and synchronizing them into the host Gimbal cluster.
 
-The Discoverer will leverage the watch feature of the Kubernetes API to receive changes dynamically, rather than having to poll the API. All available services & endpoints will be synchronized to the the same namespace matching the source system.
+The Discoverer will leverage the watch feature of the Kubernetes API to receive changes dynamically, rather than having to poll the API. All available services & endpoints will be synchronized to the same namespace matching the source system.
 
 The discoverer will only be responsible for monitoring a single cluster at a time. If multiple clusters are required to be watched, then multiple discoverers will need to be deployed.
 
@@ -26,10 +26,13 @@ Arguments are available to customize the discoverer, most have defaults but othe
 | discover-kubecfg-file | ""  | Location of kubecfg file for access to remote Kubernetes cluster to watch for services / endpoints 
 | backend-name  | ""  |   Name of cluster scraping for services & endpoints (Cannot start or end with a hyphen and must be lowercase alpha-numeric)
 | debug | false | Enable debug logging 
+| prometheus-listen-address | 8080 | The address to listen on for Prometheus HTTP requests
+| gimbal-client-qps | 5 | The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server
+| gimbal-client-burst | 10 | The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst
 
 ### Credentials
 
-A valid [Kubernetes config file](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) is required to access the remote cluster. This config file is stored as a Kubernetes secret in the Gimbal cluster. 
+A valid [Kubernetes config file](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) is required to access the remote cluster. This config file is stored as a Kubernetes secret in the Gimbal cluster.
 
 The following example creates a secret from a file locally and places it in the namespace `gimbal-discovery`. **_NOTE: Path to `config file` as well as `backend-name` will need to be customized._**
 
@@ -74,6 +77,15 @@ Credentials to the backend Kubernetes cluster can be updated at any time if nece
 4. Verify the discoverer is up and running.
 5. Delete the old secret, or rollback the deployment if the discoverer failed to start.
 
+### Configuring the Gimbal Kubernetes client rate limiting
+
+The discoverer has two configuration parameters that control the request rate limiter of the Kubernetes client used to sync services and endpoints to the Gimbal cluster:
+
+* Queries per second (QPS): Number of requests per second that can be sent to the Gimbal API server. Set using the `--gimbal-client-qps` command-line flag.
+* Burst size: Number of requests that can be sent during a burst period. A burst is a period of time in which the number of requests can exceed the configured QPS, while still maintaining a smoothed QPS rate over time. Set using the `--gimbal-client-burst` command-line flag.
+
+These configuration parameters are dependent on your requirements and the hardware running the Gimbal cluster. If services and endpoints in your environment undergo a high rate of change, increase the QPS and burst parameters, but make sure that the Gimbal API server and etcd cluster can handle the increased load.
+
 ### Data flow
 
 Data flows from the remote cluster into the Gimbal cluster. The steps on how they replicate are as follows:
@@ -85,6 +97,7 @@ Data flows from the remote cluster into the Gimbal cluster. The steps on how the
 #### Ignored Objects
 
 An exception to the flow outlined previously are objects that are ignored when synchronizing. The following rules determine if an object is ignored during sync:
+
 - Any service or endpoint in the `kube-system` namespace
 - Any service or endpoint named `kubernetes` in the `default` namespace
 

docs/openstack-discoverer.md

@@ -47,6 +47,9 @@ Arguments are available to customize the discoverer, most have defaults but othe
 | reconciliation-period | 30s | The interval of time between reconciliation loop runs 
 | http-client-timeout | 5s | The HTTP client request timeout
 | openstack-certificate-authority | "" | Path to cert file of the OpenStack API certificate authority
+| prometheus-listen-address | 8080 | The address to listen on for Prometheus HTTP requests
+| gimbal-client-qps | 5 | The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server
+| gimbal-client-burst | 10 | The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst
 
 ### Credentials
 
@@ -89,6 +92,15 @@ Credentials to the backend OpenStack cluster can be updated at any time if neces
 4. Verify the discoverer is up and running.
 5. Delete the old secret, or rollback the deployment if the discoverer failed to start.
 
+### Configuring the Gimbal Kubernetes client rate limiting
+
+The discoverer has two configuration parameters that control the request rate limiter of the Kubernetes client used to sync services and endpoints to the Gimbal cluster:
+
+* Queries per second (QPS): Number of requests per second that can be sent to the Gimbal API server. Set using the `--gimbal-client-qps` command-line flag.
+* Burst size: Number of requests that can be sent during a burst period. A burst is a period of time in which the number of requests can exceed the configured QPS, while still maintaining a smoothed QPS rate over time. Set using the `--gimbal-client-burst` command-line flag.
+
+These configuration parameters are dependent on your requirements and the hardware running the Gimbal cluster. If services and endpoints in your environment undergo a high rate of change, increase the QPS and burst parameters, but make sure that the Gimbal API server and etcd cluster can handle the increased load.
+
 ### Data flow
 
 Data flows from the remote cluster into the Gimbal cluster. The steps on how they replicate are as follows: