discovery: make gimbal client request rate limit configurable

Signed-off-by: Alexander Brand <alexbrand09@gmail.com>

discovery/cmd/kubernetes-discoverer/main.go

@@ -42,6 +42,8 @@ var (
 	debug                 bool
 	prometheusListenPort  int
 	discovererMetrics     localmetrics.DiscovererMetrics
+	gimbalKubeClientQPS   float64
+	gimbalKubeClientBurst int
 )
 
 func init() {
@@ -53,6 +55,8 @@ func init() {
 	flag.DurationVar(&resyncInterval, "resync-interval", time.Minute*30, "Default resync period for watcher to refresh")
 	flag.BoolVar(&debug, "debug", false, "Enable debug logging.")
 	flag.IntVar(&prometheusListenPort, "prometheus-listen-address", 8080, "The address to listen on for Prometheus HTTP requests")
+	flag.Float64Var(&gimbalKubeClientQPS, "gimbal-client-qps", 5, "The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server")
+	flag.IntVar(&gimbalKubeClientBurst, "gimbal-client-burst", 10, "The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst")
 	flag.Parse()
 }
 
@@ -69,6 +73,12 @@ func main() {
 	}
 
 	log.Info("Gimbal Kubernetes Discoverer Starting up...")
+	log.Infof("Version: %s", buildinfo.Version)
+	log.Infof("Backend name: %s", backendName)
+	log.Infof("Number of queue worker threads: %d", numProcessThreads)
+	log.Infof("Resync interval: %v", resyncInterval)
+	log.Infof("Gimbal kubernetes client QPS: %v", gimbalKubeClientQPS)
+	log.Infof("Gimbal kubernetes client burst: %d", gimbalKubeClientBurst)
 
 	// Init prometheus metrics
 	discovererMetrics = localmetrics.NewMetrics()
@@ -90,7 +100,7 @@ func main() {
 	}
 
 	// Init
-	gimbalKubeClient, err := k8s.NewClient(gimbalKubeCfgFile, log)
+	gimbalKubeClient, err := k8s.NewClientWithQPS(gimbalKubeCfgFile, log, float32(gimbalKubeClientQPS), gimbalKubeClientBurst)
 	if err != nil {
 		log.Fatal("Could not init k8sclient! ", err)
 	}

discovery/cmd/openstack-discoverer/main.go

@@ -50,6 +50,8 @@ var (
 	prometheusListenPort              int
 	discovererMetrics                 localmetrics.DiscovererMetrics
 	log                               *logrus.Logger
+	gimbalKubeClientQPS               float64
+	gimbalKubeClientBurst             int
 )
 
 const (
@@ -67,6 +69,8 @@ func init() {
 	flag.DurationVar(&httpClientTimeout, "http-client-timeout", 5*time.Second, "The HTTP client request timeout.")
 	flag.StringVar(&openstackCertificateAuthorityFile, "openstack-certificate-authority", "", "Path to cert file of the OpenStack API certificate authority.")
 	flag.IntVar(&prometheusListenPort, "prometheus-listen-address", 8080, "The address to listen on for Prometheus HTTP requests")
+	flag.Float64Var(&gimbalKubeClientQPS, "gimbal-client-qps", 5, "The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server")
+	flag.IntVar(&gimbalKubeClientBurst, "gimbal-client-burst", 10, "The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst")
 	flag.Parse()
 }
 
@@ -86,6 +90,12 @@ func main() {
 	}
 
 	log.Info("Gimbal OpenStack Discoverer Starting up...")
+	log.Infof("Version: %s", buildinfo.Version)
+	log.Infof("Backend name: %s", backendName)
+	log.Infof("Number of queue worker threads: %d", numProcessThreads)
+	log.Infof("Reconciliation period: %v", reconciliationPeriod)
+	log.Infof("Gimbal kubernetes client QPS: %v", gimbalKubeClientQPS)
+	log.Infof("Gimbal kubernetes client burst: %d", gimbalKubeClientBurst)
 
 	// Init prometheus metrics
 	discovererMetrics = localmetrics.NewMetrics()
@@ -97,7 +107,7 @@ func main() {
 	}
 	log.Infof("BackendName is: %s", backendName)
 
-	gimbalKubeClient, err := k8s.NewClient(gimbalKubeCfgFile, log)
+	gimbalKubeClient, err := k8s.NewClientWithQPS(gimbalKubeCfgFile, log, float32(gimbalKubeClientQPS), gimbalKubeClientBurst)
 	if err != nil {
 		log.Fatal("Failed to create kubernetes client", err)
 	}

discovery/pkg/k8s/client.go

@@ -31,6 +31,19 @@ func NewClient(kubeCfgFile string, logger *logrus.Logger) (kubernetes.Interface,
 	return kubernetes.NewForConfig(config)
 }
 
+// NewClientWithQPS returns a Kubernetes client using the given configuration
+// and rate limiting parameters. If no config is provided, assumes it is running
+// inside a Kubernetes cluster and uses the in-cluster config.
+func NewClientWithQPS(kubeCfgFile string, logger *logrus.Logger, qps float32, burst int) (kubernetes.Interface, error) {
+	config, err := buildConfig(kubeCfgFile, logger)
+	if err != nil {
+		return nil, err
+	}
+	config.QPS = qps
+	config.Burst = burst
+	return kubernetes.NewForConfig(config)
+}
+
 func buildConfig(kubeCfgFile string, logger *logrus.Logger) (*rest.Config, error) {
 	if kubeCfgFile != "" {
 		logger.Infof("Using OutOfCluster k8s config with kubeConfigFile: %s", kubeCfgFile)

docs/README.md

@@ -1,6 +1,6 @@
 # Documentation
 
-See the root-level README for an introduction, and the [deployment directory](../deployment/README.md) to get started with setting up and and deploying Gimbal.
+See the root-level README for an introduction, and the [deployment directory](../deployment/README.md) to get started with setting up and deploying Gimbal.
 
 Here you can dig into the details of how Gimbal works, and explore more advanced topics for operators and users.
 

docs/kubernetes-discoverer.md

@@ -4,7 +4,7 @@
 
 The Kubernetes discoverer provides service discovery for a Kubernetes cluster. It does this by monitoring available Services and Endpoints for a single Kubernetes cluster and synchronizing them into the host Gimbal cluster.
 
-The Discoverer will leverage the watch feature of the Kubernetes API to receive changes dynamically, rather than having to poll the API. All available services & endpoints will be synchronized to the the same namespace matching the source system.
+The Discoverer will leverage the watch feature of the Kubernetes API to receive changes dynamically, rather than having to poll the API. All available services & endpoints will be synchronized to the same namespace matching the source system.
 
 The discoverer will only be responsible for monitoring a single cluster at a time. If multiple clusters are required to be watched, then multiple discoverers will need to be deployed.
 
@@ -26,10 +26,13 @@ Arguments are available to customize the discoverer, most have defaults but othe
 | discover-kubecfg-file | ""  | Location of kubecfg file for access to remote Kubernetes cluster to watch for services / endpoints 
 | backend-name  | ""  |   Name of cluster scraping for services & endpoints (Cannot start or end with a hyphen and must be lowercase alpha-numeric)
 | debug | false | Enable debug logging 
+| prometheus-listen-address | 8080 | The address to listen on for Prometheus HTTP requests
+| gimbal-client-qps | 5 | The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server
+| gimbal-client-burst | 10 | The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst
 
 ### Credentials
 
-A valid [Kubernetes config file](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) is required to access the remote cluster. This config file is stored as a Kubernetes secret in the Gimbal cluster. 
+A valid [Kubernetes config file](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) is required to access the remote cluster. This config file is stored as a Kubernetes secret in the Gimbal cluster.
 
 The following example creates a secret from a file locally and places it in the namespace `gimbal-discovery`. **_NOTE: Path to `config file` as well as `backend-name` will need to be customized._**
 
@@ -85,6 +88,7 @@ Data flows from the remote cluster into the Gimbal cluster. The steps on how the
 #### Ignored Objects
 
 An exception to the flow outlined previously are objects that are ignored when synchronizing. The following rules determine if an object is ignored during sync:
+
 - Any service or endpoint in the `kube-system` namespace
 - Any service or endpoint named `kubernetes` in the `default` namespace
 

docs/openstack-discoverer.md

@@ -47,6 +47,9 @@ Arguments are available to customize the discoverer, most have defaults but othe
 | reconciliation-period | 30s | The interval of time between reconciliation loop runs 
 | http-client-timeout | 5s | The HTTP client request timeout
 | openstack-certificate-authority | "" | Path to cert file of the OpenStack API certificate authority
+| prometheus-listen-address | 8080 | The address to listen on for Prometheus HTTP requests
+| gimbal-client-qps | 5 | The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server
+| gimbal-client-burst | 10 | The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst
 
 ### Credentials