From 8deb87eeb126d9500be3fa2ac83dfc8709d1f37c Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 24 Apr 2018 14:56:14 -0400 Subject: [PATCH 1/6] Implement API Latency metric for Openstack Signed-off-by: Steve Sloka --- discovery/cmd/openstack-discoverer/main.go | 20 ++++++- discovery/pkg/metrics/metrics.go | 10 +++- discovery/pkg/openstack/httplogger.go | 70 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 discovery/pkg/openstack/httplogger.go diff --git a/discovery/cmd/openstack-discoverer/main.go b/discovery/cmd/openstack-discoverer/main.go index d5126c2a..1399e09f 100644 --- a/discovery/cmd/openstack-discoverer/main.go +++ b/discovery/cmd/openstack-discoverer/main.go @@ -49,6 +49,7 @@ var ( openstackCertificateAuthorityFile string prometheusListenPort int discovererMetrics localmetrics.DiscovererMetrics + log *logrus.Logger ) func init() { @@ -73,7 +74,7 @@ func main() { os.Exit(0) } - var log = logrus.New() + log = logrus.New() log.Formatter = util.GetFormatter() if debug { log.Level = logrus.DebugLevel @@ -123,6 +124,8 @@ func main() { osClient.HTTPClient.Transport = httpTransportWithCA(log, openstackCertificateAuthorityFile) } + osClient.HTTPClient = newHTTPClient() + osAuthOptions := gophercloud.AuthOptions{ IdentityEndpoint: identityEndpoint, Username: username, @@ -130,6 +133,7 @@ func main() { DomainName: "Default", TenantName: tenantName, } + if err := gopheropenstack.Authenticate(osClient, osAuthOptions); err != nil { log.Fatalf("Failed to authenticate with OpenStack: %v", err) } @@ -206,3 +210,17 @@ func httpTransportWithCA(log *logrus.Logger, caFile string) http.RoundTripper { }, } } + +// newHTTPClient return a custom HTTP client that allows for logging relevant +// information before and after the HTTP request. +func newHTTPClient() http.Client { + return http.Client{ + Transport: &openstack.LogRoundTripper{ + RoundTripper: http.DefaultTransport, + Log: log, + ClusterName: clusterName, + ClusterType: "openstack", + Metrics: &discovererMetrics, + }, + } +} diff --git a/discovery/pkg/metrics/metrics.go b/discovery/pkg/metrics/metrics.go index 4a69fadf..f646664e 100644 --- a/discovery/pkg/metrics/metrics.go +++ b/discovery/pkg/metrics/metrics.go @@ -85,7 +85,7 @@ func NewMetrics() DiscovererMetrics { Name: DiscovererAPILatencyMSGauge, Help: "The milliseconds it takes for requests to return from a remote discoverer api", }, - []string{"clustername", "clustertype"}, + []string{"clustername", "clustertype", "path"}, ) metrics[DiscovererCycleDurationMSGauge] = prometheus.NewGaugeVec( @@ -172,3 +172,11 @@ func (d *DiscovererMetrics) CycleDurationMetric(clusterName, clusterType string, m.WithLabelValues(clusterName, clusterType).Set(math.Floor(duration.Seconds() * 1e3)) } } + +// APILatencyMetric formats a cycle duration gauge prometheus metric +func (d *DiscovererMetrics) APILatencyMetric(clusterName, clusterType, path string, duration time.Duration) { + m, ok := d.metrics[DiscovererAPILatencyMSGauge].(*prometheus.GaugeVec) + if ok { + m.WithLabelValues(clusterName, clusterType, path).Set(math.Floor(duration.Seconds() * 1e3)) + } +} diff --git a/discovery/pkg/openstack/httplogger.go b/discovery/pkg/openstack/httplogger.go new file mode 100644 index 00000000..6a16bf59 --- /dev/null +++ b/discovery/pkg/openstack/httplogger.go @@ -0,0 +1,70 @@ +// Copyright © 2018 Heptio +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + "fmt" + "math" + "net/http" + "net/http/httptrace" + "time" + + localmetrics "github.com/heptio/gimbal/discovery/pkg/metrics" + "github.com/sirupsen/logrus" +) + +// LogRoundTripper satisfies the http.RoundTripper interface and is used to +// customize the default Gophercloud RoundTripper to allow for logging. +type LogRoundTripper struct { + RoundTripper http.RoundTripper + numReauthAttempts int + Log *logrus.Logger + Metrics *localmetrics.DiscovererMetrics + ClusterName string + ClusterType string +} + +// RoundTrip performs a round-trip HTTP request and logs relevant information about it. +func (lrt *LogRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + lrt.Log.Debugf("Request URL: %s", request.URL) + + start := time.Now() + var latency time.Duration + + trace := &httptrace.ClientTrace{ + GotFirstResponseByte: func() { + latency = time.Now().Sub(start) + lrt.Log.Debug("-- API Latency: ", math.Floor(latency.Seconds()*1e3)) + }, + } + request = request.WithContext(httptrace.WithClientTrace(request.Context(), trace)) + + response, err := lrt.RoundTripper.RoundTrip(request) + if response == nil { + return nil, err + } + + if response.StatusCode == http.StatusUnauthorized { + if lrt.numReauthAttempts == 3 { + return response, fmt.Errorf("Tried to re-authenticate 3 times with no success.") + } + lrt.numReauthAttempts++ + } + + lrt.Log.Debugf("-- Response Status: %s", response.Status) + + lrt.Metrics.APILatencyMetric(lrt.ClusterName, lrt.ClusterType, request.URL.Path, latency) + + return response, nil +} From 51d4fe84382196ea1adcece425b71786a3c0b478 Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 24 Apr 2018 14:59:53 -0400 Subject: [PATCH 2/6] Update docs with API Latency request change Signed-off-by: Steve Sloka --- docs/monitoring.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/monitoring.md b/docs/monitoring.md index c8072f5b..deae8653 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -94,6 +94,7 @@ The Gimbal Discoverer currently has two different systems it can monitor, Kubern - **gimbal_discoverer_api_latency_ms (gauge):** The milliseconds it takes for requests to return from a remote discoverer api (e.g. Openstack) - clustername - clustertype + - path: API request path - **gimbal_discoverer_cycle_duration_ms (gauge):** The milliseconds it takes for all objects to be synced from a remote discoverer api (e.g. Openstack) - clustername - clustertype From d4a8c6052005c829415f196f0811bedbae9540c9 Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 24 Apr 2018 16:06:19 -0400 Subject: [PATCH 3/6] Refactor http client / move clusterType to main Signed-off-by: Steve Sloka --- discovery/cmd/openstack-discoverer/main.go | 20 +++++++++++++++++--- discovery/pkg/openstack/reconciler.go | 6 +++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/discovery/cmd/openstack-discoverer/main.go b/discovery/cmd/openstack-discoverer/main.go index 1399e09f..7826087c 100644 --- a/discovery/cmd/openstack-discoverer/main.go +++ b/discovery/cmd/openstack-discoverer/main.go @@ -52,6 +52,8 @@ var ( log *logrus.Logger ) +const clusterType = "openstack" + func init() { flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.StringVar(&gimbalKubeCfgFile, "gimbal-kubecfg-file", "", "Location of kubecfg file for access to gimbal system kubernetes api, defaults to service account tokens") @@ -119,12 +121,24 @@ func main() { if err != nil { log.Fatalf("Failed to create OpenStack client: %v", err) } - osClient.HTTPClient.Timeout = httpClientTimeout + + dClient := http.Client{ + Transport: &openstack.LogRoundTripper{ + RoundTripper: http.DefaultTransport, + Log: log, + ClusterName: clusterName, + ClusterType: clusterType, + Metrics: &discovererMetrics, + }, + Timeout: httpClientTimeout, + } + if openstackCertificateAuthorityFile != "" { - osClient.HTTPClient.Transport = httpTransportWithCA(log, openstackCertificateAuthorityFile) + dClient.Transport = httpTransportWithCA(log, openstackCertificateAuthorityFile) } - osClient.HTTPClient = newHTTPClient() + // Pass http client + osClient.HTTPClient = dClient osAuthOptions := gophercloud.AuthOptions{ IdentityEndpoint: identityEndpoint, diff --git a/discovery/pkg/openstack/reconciler.go b/discovery/pkg/openstack/reconciler.go index 3fc250ad..0c524e84 100644 --- a/discovery/pkg/openstack/reconciler.go +++ b/discovery/pkg/openstack/reconciler.go @@ -29,8 +29,6 @@ import ( "k8s.io/client-go/kubernetes" ) -const clusterType = "openstack" - type ProjectLister interface { ListProjects() ([]projects.Project, error) } @@ -50,6 +48,7 @@ type Reconciler struct { // ClusterName is the name of the OpenStack cluster ClusterName string + ClusterType string // GimbalKubeClient is the client of the Kubernetes cluster where Gimbal is running GimbalKubeClient kubernetes.Interface // Interval between reconciliation loops @@ -61,10 +60,11 @@ type Reconciler struct { } // NewReconciler returns an OpenStack reconciler -func NewReconciler(clusterName string, gimbalKubeClient kubernetes.Interface, syncPeriod time.Duration, lbLister LoadBalancerLister, +func NewReconciler(clusterName, clusterType string, gimbalKubeClient kubernetes.Interface, syncPeriod time.Duration, lbLister LoadBalancerLister, projectLister ProjectLister, log *logrus.Logger, queueWorkers int, metrics localmetrics.DiscovererMetrics) Reconciler { return Reconciler{ ClusterName: clusterName, + ClusterType: clusterType, GimbalKubeClient: gimbalKubeClient, SyncPeriod: syncPeriod, LoadBalancerLister: lbLister, From 460796d419c6123ac5b6af9d28f33f687af02a2e Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 24 Apr 2018 16:07:03 -0400 Subject: [PATCH 4/6] Remove unneeded code Signed-off-by: Steve Sloka --- discovery/cmd/openstack-discoverer/main.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/discovery/cmd/openstack-discoverer/main.go b/discovery/cmd/openstack-discoverer/main.go index 7826087c..a8783573 100644 --- a/discovery/cmd/openstack-discoverer/main.go +++ b/discovery/cmd/openstack-discoverer/main.go @@ -224,17 +224,3 @@ func httpTransportWithCA(log *logrus.Logger, caFile string) http.RoundTripper { }, } } - -// newHTTPClient return a custom HTTP client that allows for logging relevant -// information before and after the HTTP request. -func newHTTPClient() http.Client { - return http.Client{ - Transport: &openstack.LogRoundTripper{ - RoundTripper: http.DefaultTransport, - Log: log, - ClusterName: clusterName, - ClusterType: "openstack", - Metrics: &discovererMetrics, - }, - } -} From 074208fe838c2816e0f1debb6c6ba45f70469fd7 Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 24 Apr 2018 16:09:32 -0400 Subject: [PATCH 5/6] Fix tests / Fix passing clustertype Signed-off-by: Steve Sloka --- discovery/cmd/openstack-discoverer/main.go | 1 + discovery/pkg/openstack/reconciler.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/discovery/cmd/openstack-discoverer/main.go b/discovery/cmd/openstack-discoverer/main.go index a8783573..4ef56665 100644 --- a/discovery/cmd/openstack-discoverer/main.go +++ b/discovery/cmd/openstack-discoverer/main.go @@ -164,6 +164,7 @@ func main() { reconciler := openstack.NewReconciler( clusterName, + clusterType, gimbalKubeClient, reconciliationPeriod, lbv2, diff --git a/discovery/pkg/openstack/reconciler.go b/discovery/pkg/openstack/reconciler.go index 0c524e84..ab64ed73 100644 --- a/discovery/pkg/openstack/reconciler.go +++ b/discovery/pkg/openstack/reconciler.go @@ -155,7 +155,7 @@ func (r *Reconciler) reconcile() { } // Log to Prometheus the cycle duration - r.Metrics.CycleDurationMetric(r.ClusterName, clusterType, time.Now().Sub(start)) + r.Metrics.CycleDurationMetric(r.ClusterName, r.ClusterType, time.Now().Sub(start)) } func (r *Reconciler) reconcileSvcs(desiredSvcs, currentSvcs []v1.Service) { From 132298722782e75458c0385ef26f97a070bf84d1 Mon Sep 17 00:00:00 2001 From: Steve Sloka Date: Tue, 24 Apr 2018 16:34:32 -0400 Subject: [PATCH 6/6] Refactor http transport Signed-off-by: Steve Sloka --- discovery/cmd/openstack-discoverer/main.go | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/discovery/cmd/openstack-discoverer/main.go b/discovery/cmd/openstack-discoverer/main.go index 4ef56665..b19c316d 100644 --- a/discovery/cmd/openstack-discoverer/main.go +++ b/discovery/cmd/openstack-discoverer/main.go @@ -122,23 +122,22 @@ func main() { log.Fatalf("Failed to create OpenStack client: %v", err) } - dClient := http.Client{ - Transport: &openstack.LogRoundTripper{ - RoundTripper: http.DefaultTransport, - Log: log, - ClusterName: clusterName, - ClusterType: clusterType, - Metrics: &discovererMetrics, - }, - Timeout: httpClientTimeout, + transport := &openstack.LogRoundTripper{ + RoundTripper: http.DefaultTransport, + Log: log, + ClusterName: clusterName, + ClusterType: clusterType, + Metrics: &discovererMetrics, } if openstackCertificateAuthorityFile != "" { - dClient.Transport = httpTransportWithCA(log, openstackCertificateAuthorityFile) + transport.RoundTripper = httpTransportWithCA(log, openstackCertificateAuthorityFile) } - // Pass http client - osClient.HTTPClient = dClient + osClient.HTTPClient = http.Client{ + Transport: transport, + Timeout: httpClientTimeout, + } osAuthOptions := gophercloud.AuthOptions{ IdentityEndpoint: identityEndpoint,