Merge pull request #392 from alphagov/route-reload-duration-metrics

Add a summary metric for route reload durations

lib/metrics.go

@@ -15,18 +15,18 @@ var (
  []string{"host"},
  )
 
- routeReloadCountMetric = prometheus.NewCounter(
- prometheus.CounterOpts{
- Name: "router_route_reload_total",
- Help: "Total number of attempts to reload the routing table",
- },
- )
-
- routeReloadErrorCountMetric = prometheus.NewCounter(
- prometheus.CounterOpts{
- Name: "router_route_reload_error_total",
- Help: "Number of failed attempts to reload the routing table",
+ routeReloadDurationMetric = prometheus.NewSummaryVec(
+ prometheus.SummaryOpts{
+ Name: "router_route_reload_duration_seconds",
+ Help: "Histogram of route reload durations in seconds",
+ Objectives: map[float64]float64{
+ 0.5: 0.01,
+ 0.9: 0.01,
+ 0.95: 0.01,
+ 0.99: 0.005,
+ },
  },
+ []string{"success"},
  )
 
  routesCountMetric = prometheus.NewGauge(
@@ -40,8 +40,7 @@ var (
 func registerMetrics(r prometheus.Registerer) {
  r.MustRegister(
  internalServerErrorCountMetric,
- routeReloadCountMetric,
- routeReloadErrorCountMetric,
+ routeReloadDurationMetric,
  routesCountMetric,
  )
  handlers.RegisterMetrics(r)

lib/router.go

@@ -5,6 +5,7 @@ import (
  "net/http"
  "net/url"
  "os"
+ "strconv"
  "sync"
  "time"
 
@@ -213,21 +214,24 @@ type mongoDatabase interface {
 // create a new proxy mux, load applications (backends) and routes into it, and
 // then flip the "mux" pointer in the Router.
 func (rt *Router) reloadRoutes(db *mgo.Database, currentOptime bson.MongoTimestamp) {
+ var success bool
+ timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
+ labels := prometheus.Labels{"success": strconv.FormatBool(success)}
+ routeReloadDurationMetric.With(labels).Observe(v)
+ }))
  defer func() {
- // increment this metric regardless of whether the route reload succeeded
- routeReloadCountMetric.Inc()
-
+ success = true
  if r := recover(); r != nil {
+ success = false
  logWarn("router: recovered from panic in reloadRoutes:", r)
  logInfo("router: original routes have not been modified")
  errorMessage := fmt.Sprintf("panic: %v", r)
  err := logger.RecoveredError{ErrorMessage: errorMessage}
  logger.NotifySentry(logger.ReportableError{Error: err})
-
- routeReloadErrorCountMetric.Inc()
  } else {
  rt.mongoReadToOptime = currentOptime
  }
+ timer.ObserveDuration()
  }()
 
  logInfo("router: reloading routes")