From 949f11010b67c14365f908c18ec778c86014fbf8 Mon Sep 17 00:00:00 2001
From: Alexander Brand <alexbrand09@gmail.com>
Date: Mon, 11 Jun 2018 10:33:18 -0400
Subject: [PATCH 1/2] discovery: make gimbal client request rate limit
configurable
Signed-off-by: Alexander Brand <alexbrand09@gmail.com>
---
discovery/cmd/kubernetes-discoverer/main.go | 12 +++++++++++-
discovery/cmd/openstack-discoverer/main.go | 12 +++++++++++-
discovery/pkg/k8s/client.go | 13 +++++++++++++
docs/README.md | 2 +-
docs/kubernetes-discoverer.md | 8 ++++++--
docs/openstack-discoverer.md | 3 +++
6 files changed, 45 insertions(+), 5 deletions(-)
diff --git a/discovery/cmd/kubernetes-discoverer/main.go b/discovery/cmd/kubernetes-discoverer/main.go
index f40d6592..aae1fdf4 100644
--- a/discovery/cmd/kubernetes-discoverer/main.go
+++ b/discovery/cmd/kubernetes-discoverer/main.go
@@ -42,6 +42,8 @@ var (
debug bool
prometheusListenPort int
discovererMetrics localmetrics.DiscovererMetrics
+ gimbalKubeClientQPS float64
+ gimbalKubeClientBurst int
)
func init() {
@@ -53,6 +55,8 @@ func init() {
flag.DurationVar(&resyncInterval, "resync-interval", time.Minute*30, "Default resync period for watcher to refresh")
flag.BoolVar(&debug, "debug", false, "Enable debug logging.")
flag.IntVar(&prometheusListenPort, "prometheus-listen-address", 8080, "The address to listen on for Prometheus HTTP requests")
+ flag.Float64Var(&gimbalKubeClientQPS, "gimbal-client-qps", 5, "The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server")
+ flag.IntVar(&gimbalKubeClientBurst, "gimbal-client-burst", 10, "The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst")
flag.Parse()
}
@@ -69,6 +73,12 @@ func main() {
}
log.Info("Gimbal Kubernetes Discoverer Starting up...")
+ log.Infof("Version: %s", buildinfo.Version)
+ log.Infof("Backend name: %s", backendName)
+ log.Infof("Number of queue worker threads: %d", numProcessThreads)
+ log.Infof("Resync interval: %v", resyncInterval)
+ log.Infof("Gimbal kubernetes client QPS: %v", gimbalKubeClientQPS)
+ log.Infof("Gimbal kubernetes client burst: %d", gimbalKubeClientBurst)
// Init prometheus metrics
discovererMetrics = localmetrics.NewMetrics()
@@ -90,7 +100,7 @@ func main() {
}
// Init
- gimbalKubeClient, err := k8s.NewClient(gimbalKubeCfgFile, log)
+ gimbalKubeClient, err := k8s.NewClientWithQPS(gimbalKubeCfgFile, log, float32(gimbalKubeClientQPS), gimbalKubeClientBurst)
if err != nil {
log.Fatal("Could not init k8sclient! ", err)
}
diff --git a/discovery/cmd/openstack-discoverer/main.go b/discovery/cmd/openstack-discoverer/main.go
index b1df8489..d76a8090 100644
--- a/discovery/cmd/openstack-discoverer/main.go
+++ b/discovery/cmd/openstack-discoverer/main.go
@@ -50,6 +50,8 @@ var (
prometheusListenPort int
discovererMetrics localmetrics.DiscovererMetrics
log *logrus.Logger
+ gimbalKubeClientQPS float64
+ gimbalKubeClientBurst int
)
const (
@@ -67,6 +69,8 @@ func init() {
flag.DurationVar(&httpClientTimeout, "http-client-timeout", 5*time.Second, "The HTTP client request timeout.")
flag.StringVar(&openstackCertificateAuthorityFile, "openstack-certificate-authority", "", "Path to cert file of the OpenStack API certificate authority.")
flag.IntVar(&prometheusListenPort, "prometheus-listen-address", 8080, "The address to listen on for Prometheus HTTP requests")
+ flag.Float64Var(&gimbalKubeClientQPS, "gimbal-client-qps", 5, "The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server")
+ flag.IntVar(&gimbalKubeClientBurst, "gimbal-client-burst", 10, "The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst")
flag.Parse()
}
@@ -86,6 +90,12 @@ func main() {
}
log.Info("Gimbal OpenStack Discoverer Starting up...")
+ log.Infof("Version: %s", buildinfo.Version)
+ log.Infof("Backend name: %s", backendName)
+ log.Infof("Number of queue worker threads: %d", numProcessThreads)
+ log.Infof("Reconciliation period: %v", reconciliationPeriod)
+ log.Infof("Gimbal kubernetes client QPS: %v", gimbalKubeClientQPS)
+ log.Infof("Gimbal kubernetes client burst: %d", gimbalKubeClientBurst)
// Init prometheus metrics
discovererMetrics = localmetrics.NewMetrics()
@@ -97,7 +107,7 @@ func main() {
}
log.Infof("BackendName is: %s", backendName)
- gimbalKubeClient, err := k8s.NewClient(gimbalKubeCfgFile, log)
+ gimbalKubeClient, err := k8s.NewClientWithQPS(gimbalKubeCfgFile, log, float32(gimbalKubeClientQPS), gimbalKubeClientBurst)
if err != nil {
log.Fatal("Failed to create kubernetes client", err)
}
diff --git a/discovery/pkg/k8s/client.go b/discovery/pkg/k8s/client.go
index c4f7790d..38054fe3 100644
--- a/discovery/pkg/k8s/client.go
+++ b/discovery/pkg/k8s/client.go
@@ -31,6 +31,19 @@ func NewClient(kubeCfgFile string, logger *logrus.Logger) (kubernetes.Interface,
return kubernetes.NewForConfig(config)
}
+// NewClientWithQPS returns a Kubernetes client using the given configuration
+// and rate limiting parameters. If no config is provided, assumes it is running
+// inside a Kubernetes cluster and uses the in-cluster config.
+func NewClientWithQPS(kubeCfgFile string, logger *logrus.Logger, qps float32, burst int) (kubernetes.Interface, error) {
+ config, err := buildConfig(kubeCfgFile, logger)
+ if err != nil {
+ return nil, err
+ }
+ config.QPS = qps
+ config.Burst = burst
+ return kubernetes.NewForConfig(config)
+}
+
func buildConfig(kubeCfgFile string, logger *logrus.Logger) (*rest.Config, error) {
if kubeCfgFile != "" {
logger.Infof("Using OutOfCluster k8s config with kubeConfigFile: %s", kubeCfgFile)
diff --git a/docs/README.md b/docs/README.md
index 1ae47556..5059e671 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,6 @@
# Documentation
-See the root-level README for an introduction, and the [deployment directory](../deployment/README.md) to get started with setting up and and deploying Gimbal.
+See the root-level README for an introduction, and the [deployment directory](../deployment/README.md) to get started with setting up and deploying Gimbal.
Here you can dig into the details of how Gimbal works, and explore more advanced topics for operators and users.
diff --git a/docs/kubernetes-discoverer.md b/docs/kubernetes-discoverer.md
index 7410f6f3..835a44f3 100644
--- a/docs/kubernetes-discoverer.md
+++ b/docs/kubernetes-discoverer.md
@@ -4,7 +4,7 @@
The Kubernetes discoverer provides service discovery for a Kubernetes cluster. It does this by monitoring available Services and Endpoints for a single Kubernetes cluster and synchronizing them into the host Gimbal cluster.
-The Discoverer will leverage the watch feature of the Kubernetes API to receive changes dynamically, rather than having to poll the API. All available services & endpoints will be synchronized to the the same namespace matching the source system.
+The Discoverer will leverage the watch feature of the Kubernetes API to receive changes dynamically, rather than having to poll the API. All available services & endpoints will be synchronized to the same namespace matching the source system.
The discoverer will only be responsible for monitoring a single cluster at a time. If multiple clusters are required to be watched, then multiple discoverers will need to be deployed.
@@ -26,10 +26,13 @@ Arguments are available to customize the discoverer, most have defaults but othe
| discover-kubecfg-file | "" | Location of kubecfg file for access to remote Kubernetes cluster to watch for services / endpoints
| backend-name | "" | Name of cluster scraping for services & endpoints (Cannot start or end with a hyphen and must be lowercase alpha-numeric)
| debug | false | Enable debug logging
+| prometheus-listen-address | 8080 | The address to listen on for Prometheus HTTP requests
+| gimbal-client-qps | 5 | The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server
+| gimbal-client-burst | 10 | The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst
### Credentials
-A valid [Kubernetes config file](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) is required to access the remote cluster. This config file is stored as a Kubernetes secret in the Gimbal cluster.
+A valid [Kubernetes config file](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) is required to access the remote cluster. This config file is stored as a Kubernetes secret in the Gimbal cluster.
The following example creates a secret from a file locally and places it in the namespace `gimbal-discovery`. **_NOTE: Path to `config file` as well as `backend-name` will need to be customized._**
@@ -85,6 +88,7 @@ Data flows from the remote cluster into the Gimbal cluster. The steps on how the
#### Ignored Objects
An exception to the flow outlined previously are objects that are ignored when synchronizing. The following rules determine if an object is ignored during sync:
+
- Any service or endpoint in the `kube-system` namespace
- Any service or endpoint named `kubernetes` in the `default` namespace
diff --git a/docs/openstack-discoverer.md b/docs/openstack-discoverer.md
index e9114b0c..99b86b1d 100644
--- a/docs/openstack-discoverer.md
+++ b/docs/openstack-discoverer.md
@@ -47,6 +47,9 @@ Arguments are available to customize the discoverer, most have defaults but othe
| reconciliation-period | 30s | The interval of time between reconciliation loop runs
| http-client-timeout | 5s | The HTTP client request timeout
| openstack-certificate-authority | "" | Path to cert file of the OpenStack API certificate authority
+| prometheus-listen-address | 8080 | The address to listen on for Prometheus HTTP requests
+| gimbal-client-qps | 5 | The maximum queries per second (QPS) that can be performed on the Gimbal Kubernetes API server
+| gimbal-client-burst | 10 | The maximum number of queries that can be performed on the Gimbal Kubernetes API server during a burst
### Credentials
From c738d2477d8a555f069b20709c48aca37c9844dd Mon Sep 17 00:00:00 2001
From: Alexander Brand <alexbrand09@gmail.com>
Date: Thu, 14 Jun 2018 14:03:45 -0700
Subject: [PATCH 2/2] Add documentation around qps and burst paramters
Signed-off-by: Alexander Brand <alexbrand09@gmail.com>
---
docs/kubernetes-discoverer.md | 9 +++++++++
docs/openstack-discoverer.md | 9 +++++++++
2 files changed, 18 insertions(+)
diff --git a/docs/kubernetes-discoverer.md b/docs/kubernetes-discoverer.md
index 835a44f3..893affd1 100644
--- a/docs/kubernetes-discoverer.md
+++ b/docs/kubernetes-discoverer.md
@@ -77,6 +77,15 @@ Credentials to the backend Kubernetes cluster can be updated at any time if nece
4. Verify the discoverer is up and running.
5. Delete the old secret, or rollback the deployment if the discoverer failed to start.
+### Configuring the Gimbal Kubernetes client rate limiting
+
+The discoverer has two configuration parameters that control the request rate limiter of the Kubernetes client used to sync services and endpoints to the Gimbal cluster:
+
+* Queries per second (QPS): Number of requests per second that can be sent to the Gimbal API server. Set using the `--gimbal-client-qps` command-line flag.
+* Burst size: Number of requests that can be sent during a burst period. A burst is a period of time in which the number of requests can exceed the configured QPS, while still maintaining a smoothed QPS rate over time. Set using the `--gimbal-client-burst` command-line flag.
+
+These configuration parameters are dependent on your requirements and the hardware running the Gimbal cluster. If services and endpoints in your environment undergo a high rate of change, increase the QPS and burst parameters, but make sure that the Gimbal API server and etcd cluster can handle the increased load.
+
### Data flow
Data flows from the remote cluster into the Gimbal cluster. The steps on how they replicate are as follows:
diff --git a/docs/openstack-discoverer.md b/docs/openstack-discoverer.md
index 99b86b1d..d0151e21 100644
--- a/docs/openstack-discoverer.md
+++ b/docs/openstack-discoverer.md
@@ -92,6 +92,15 @@ Credentials to the backend OpenStack cluster can be updated at any time if neces
4. Verify the discoverer is up and running.
5. Delete the old secret, or rollback the deployment if the discoverer failed to start.
+### Configuring the Gimbal Kubernetes client rate limiting
+
+The discoverer has two configuration parameters that control the request rate limiter of the Kubernetes client used to sync services and endpoints to the Gimbal cluster:
+
+* Queries per second (QPS): Number of requests per second that can be sent to the Gimbal API server. Set using the `--gimbal-client-qps` command-line flag.
+* Burst size: Number of requests that can be sent during a burst period. A burst is a period of time in which the number of requests can exceed the configured QPS, while still maintaining a smoothed QPS rate over time. Set using the `--gimbal-client-burst` command-line flag.
+
+These configuration parameters are dependent on your requirements and the hardware running the Gimbal cluster. If services and endpoints in your environment undergo a high rate of change, increase the QPS and burst parameters, but make sure that the Gimbal API server and etcd cluster can handle the increased load.
+
### Data flow
Data flows from the remote cluster into the Gimbal cluster. The steps on how they replicate are as follows: