From 004e04a1d1882f3d1258db8114a49866a420abc7 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Sun, 7 Oct 2018 17:15:12 -0700 Subject: [PATCH 1/3] etcdserver/api/etcdhttp: add "etcd_server_health_success/failures" Signed-off-by: Gyuho Lee --- etcdserver/api/etcdhttp/metrics.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/etcdserver/api/etcdhttp/metrics.go b/etcdserver/api/etcdhttp/metrics.go index bec5e9e0549..4532d32c628 100644 --- a/etcdserver/api/etcdhttp/metrics.go +++ b/etcdserver/api/etcdhttp/metrics.go @@ -24,6 +24,7 @@ import ( "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/raft" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -67,6 +68,26 @@ func NewHealthHandler(hfunc func() Health) http.HandlerFunc { } } +var ( + healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "health_success", + Help: "The total number of successful health checks", + }) + healthFailed = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "health_failures", + Help: "The total number of failed health checks", + }) +) + +func init() { + prometheus.MustRegister(healthSuccess) + prometheus.MustRegister(healthFailed) +} + // Health defines etcd server health status. // TODO: remove manual parsing in etcdctl cluster-health type Health struct { @@ -97,5 +118,11 @@ func checkHealth(srv etcdserver.ServerV2) Health { h.Health = "false" } } + + if h.Health == "true" { + healthSuccess.Inc() + } else { + healthFailed.Inc() + } return h } From 601d8b46770b8c7d6b3bb5e734133225192491ba Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Sun, 7 Oct 2018 17:16:18 -0700 Subject: [PATCH 2/3] etcdserver/api/etcdhttp: remove unused "HandleHealth" function Signed-off-by: Gyuho Lee --- etcdserver/api/etcdhttp/metrics.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/etcdserver/api/etcdhttp/metrics.go b/etcdserver/api/etcdhttp/metrics.go index 4532d32c628..4d058e46719 100644 --- a/etcdserver/api/etcdhttp/metrics.go +++ b/etcdserver/api/etcdhttp/metrics.go @@ -44,11 +44,6 @@ func HandlePrometheus(mux *http.ServeMux) { mux.Handle(pathMetrics, promhttp.Handler()) } -// HandleHealth registers health handler on '/health'. -func HandleHealth(mux *http.ServeMux, srv etcdserver.ServerV2) { - mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) })) -} - // NewHealthHandler handles '/health' requests. func NewHealthHandler(hfunc func() Health) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { From 7524cc6f4c99fe0be887c8e44cad9069c659697e Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Sun, 7 Oct 2018 17:25:14 -0700 Subject: [PATCH 3/3] integration: add "TestMetricsHealth" Signed-off-by: Gyuho Lee --- integration/metrics_test.go | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/integration/metrics_test.go b/integration/metrics_test.go index 70dcf7b0356..207e437cb6e 100644 --- a/integration/metrics_test.go +++ b/integration/metrics_test.go @@ -16,14 +16,15 @@ package integration import ( "context" + "net/http" "strconv" "testing" "time" "go.etcd.io/etcd/etcdserver" - pb "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/pkg/testutil" + "go.etcd.io/etcd/pkg/transport" ) // TestMetricDbSizeBoot checks that the db size metric is set on boot. @@ -165,3 +166,33 @@ func TestMetricQuotaBackendBytes(t *testing.T) { t.Fatalf("expected %d, got %f", etcdserver.DefaultQuotaBytes, qv) } } + +func TestMetricsHealth(t *testing.T) { + defer testutil.AfterTest(t) + clus := NewClusterV3(t, &ClusterConfig{Size: 1}) + defer clus.Terminate(t) + + tr, err := transport.NewTransport(transport.TLSInfo{}, 5*time.Second) + if err != nil { + t.Fatal(err) + } + u := clus.Members[0].ClientURLs[0] + u.Path = "/health" + resp, err := tr.RoundTrip(&http.Request{ + Header: make(http.Header), + Method: http.MethodGet, + URL: &u, + }) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + + hv, err := clus.Members[0].Metric("etcd_server_health_success") + if err != nil { + t.Fatal(err) + } + if hv != "1" { + t.Fatalf("expected '1' from /health, got %q", hv) + } +}