Skip to content

Commit

Permalink
*: Change gRPC proxy to expose etcd server endpoint /metrics
Browse files Browse the repository at this point in the history
This PR resolves an issue where the `/metrics` endpoints exposed by the proxy were not returning metrics of the etcd members servers but of the proxy itself.

Signed-off-by: Sam Batschelet <sbatsche@redhat.com>
  • Loading branch information
hexfusion committed Apr 11, 2019
1 parent ad5e169 commit 5649875
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 11 deletions.
25 changes: 25 additions & 0 deletions Documentation/op-guide/grpc_proxy.md
Original file line number Diff line number Diff line change
Expand Up @@ -223,3 +223,28 @@ Finally, test the TLS termination by putting a key into the proxy over http:
$ ETCDCTL_API=3 etcdctl --endpoints=http://localhost:12379 put abc def
# OK
```

## Metrics and Health

The gRPC proxy exposes `/health` and Prometheus `/metrics` endpoints for the etcd members defined by `--endpoints`. An alternative define an additional URL that will respond to both the `/metrics` and `/health` endpoints with the `--metrics-addr` flag.

```bash
$ etcd grpc-proxy start \
--endpoints https://localhost:2379 \
--metrics-addr https://0.0.0.0:4443 \
--listen-addr 127.0.0.1:23790 \
--key client.key \
--key-file proxy-server.key \
--cert client.crt \
--cert-file proxy-server.crt \
--cacert ca.pem \
--trusted-ca-file proxy-ca.pem
```

### Known issue

The main interface of the proxy serves both HTTP2 and HTTP/1.1. If proxy is setup with TLS as show in the above example, when using a client such as cURL against the listening interface will require explicitly setting the protocol to HTTP/1.1 on the request to return `/metrics` or `/health`. By using the `--metrics-addr` flag the secondary interface will not have this requirement.

```bash
$ curl --cacert proxy-ca.pem --key proxy-client.key --cert proxy-client.crt https://127.0.0.1:23790/metrics --http1.1
```
51 changes: 44 additions & 7 deletions etcdmain/grpc_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ package etcdmain

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"math"
Expand All @@ -30,7 +32,6 @@ import (
"github.com/coreos/etcd/clientv3/leasing"
"github.com/coreos/etcd/clientv3/namespace"
"github.com/coreos/etcd/clientv3/ordering"
"github.com/coreos/etcd/etcdserver/api/etcdhttp"
"github.com/coreos/etcd/etcdserver/api/v3election/v3electionpb"
"github.com/coreos/etcd/etcdserver/api/v3lock/v3lockpb"
pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
Expand Down Expand Up @@ -110,7 +111,7 @@ func newGRPCProxyStartCommand() *cobra.Command {

cmd.Flags().StringVar(&grpcProxyListenAddr, "listen-addr", "127.0.0.1:23790", "listen address")
cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster")
cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for /metrics requests on an additional interface")
cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for endpoint /metrics requests on an additional interface")
cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records")
cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints")
cmd.Flags().StringVar(&grpcProxyAdvertiseClientURL, "advertise-client-url", "127.0.0.1:23790", "advertise address to register (must be reachable by client)")
Expand Down Expand Up @@ -180,6 +181,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
}()

client := mustNewClient()
httpClient := mustNewHTTPClient()

srvhttp, httpl := mustHTTPListener(m, tlsinfo, client)
errc := make(chan error)
Expand All @@ -190,7 +192,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
mhttpl := mustMetricsListener(tlsinfo)
go func() {
mux := http.NewServeMux()
etcdhttp.HandlePrometheus(mux)
grpcproxy.HandleMetrics(mux, httpClient, client.Endpoints())
grpcproxy.HandleHealth(mux, client)
plog.Fatal(http.Serve(mhttpl, mux))
}()
Expand Down Expand Up @@ -352,16 +354,14 @@ func newGRPCProxyServer(client *clientv3.Client) *grpc.Server {
v3electionpb.RegisterElectionServer(server, electionp)
v3lockpb.RegisterLockServer(server, lockp)

// set zero values for metrics registered for this grpc server
grpc_prometheus.Register(server)

return server
}

func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo, c *clientv3.Client) (*http.Server, net.Listener) {
httpClient := mustNewHTTPClient()
httpmux := http.NewServeMux()
httpmux.HandleFunc("/", http.NotFound)
etcdhttp.HandlePrometheus(httpmux)
grpcproxy.HandleMetrics(httpmux, httpClient, c.Endpoints())
grpcproxy.HandleHealth(httpmux, c)
if grpcProxyEnablePprof {
for p, h := range debugutil.PProfHandlers() {
Expand All @@ -383,6 +383,43 @@ func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo, c *clientv3.Clien
return srvhttp, m.Match(cmux.Any())
}

func mustNewHTTPClient() *http.Client {
transport, err := newHTTPTransport(grpcProxyCA, grpcProxyCert, grpcProxyKey)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return &http.Client{Transport: transport}
}

func newHTTPTransport(ca, cert, key string) (*http.Transport, error) {
tr := &http.Transport{}

if ca != "" && cert != "" && key != "" {
caCert, err := ioutil.ReadFile(ca)
if err != nil {
return nil, err
}
keyPair, err := tls.LoadX509KeyPair(cert, key)
if err != nil {
return nil, err
}
caPool := x509.NewCertPool()
caPool.AppendCertsFromPEM(caCert)

tlsConfig := &tls.Config{
Certificates: []tls.Certificate{keyPair},
RootCAs: caPool,
}
tlsConfig.BuildNameToCertificate()
tr.TLSClientConfig = tlsConfig
} else if grpcProxyInsecureSkipTLSVerify {
tlsConfig := &tls.Config{InsecureSkipVerify: grpcProxyInsecureSkipTLSVerify}
tr.TLSClientConfig = tlsConfig
}
return tr, nil
}

func mustMetricsListener(tlsinfo *transport.TLSInfo) net.Listener {
murl, err := url.Parse(grpcProxyMetricsListenAddr)
if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions etcdserver/api/etcdhttp/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,19 @@ import (
)

const (
pathMetrics = "/metrics"
PathMetrics = "/metrics"
PathHealth = "/health"
)

// HandleMetricsHealth registers metrics and health handlers.
func HandleMetricsHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
mux.Handle(pathMetrics, promhttp.Handler())
mux.Handle(PathMetrics, promhttp.Handler())
mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
}

// HandlePrometheus registers prometheus handler on '/metrics'.
func HandlePrometheus(mux *http.ServeMux) {
mux.Handle(pathMetrics, promhttp.Handler())
mux.Handle(PathMetrics, promhttp.Handler())
}

// NewHealthHandler handles '/health' requests.
Expand Down
58 changes: 57 additions & 1 deletion proxy/grpcproxy/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,17 @@

package grpcproxy

import "github.com/prometheus/client_golang/prometheus"
import (
"fmt"
"io/ioutil"
"math/rand"
"net/http"
"strings"
"time"

"github.com/coreos/etcd/etcdserver/api/etcdhttp"
"github.com/prometheus/client_golang/prometheus"
)

var (
watchersCoalescing = prometheus.NewGauge(prometheus.GaugeOpts{
Expand Down Expand Up @@ -56,3 +66,49 @@ func init() {
prometheus.MustRegister(cacheHits)
prometheus.MustRegister(cachedMisses)
}

// HandleMetrics performs a GET request against etcd endpoint and returns '/metrics'.
func HandleMetrics(mux *http.ServeMux, c *http.Client, eps []string) {
// random shuffle endpoints
r := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
if len(eps) > 1 {
eps = shuffleEndpoints(r, eps)
}

pathMetrics := etcdhttp.PathMetrics
mux.HandleFunc(pathMetrics, func(w http.ResponseWriter, r *http.Request) {
target := fmt.Sprintf("%s%s", eps[0], pathMetrics)
if !strings.HasPrefix(target, "http") {
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
target = fmt.Sprintf("%s://%s", scheme, target)
}

resp, err := c.Get(target)
if err != nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
body, _ := ioutil.ReadAll(resp.Body)
fmt.Fprintf(w, "%s", body)
})
}

func shuffleEndpoints(r *rand.Rand, eps []string) []string {
// copied from Go 1.9<= rand.Rand.Perm
n := len(eps)
p := make([]int, n)
for i := 0; i < n; i++ {
j := r.Intn(i + 1)
p[i] = p[j]
p[j] = i
}
neps := make([]string, n)
for i, k := range p {
neps[i] = eps[k]
}
return neps
}

0 comments on commit 5649875

Please sign in to comment.