IDM-410: Prometheus Metrics for the Secure token Service (#2)

* health: Adds new prometheus metrics and metrics endpoint (ory#827)

Signed-off-by: Dmitry Dolbik <dolbik@gmail.com>

* IDM-410 Prometheus Metrics for the Secure token Service

Gopkg.toml

@@ -148,3 +148,7 @@
 [[constraint]]
   name = "github.com/golang/mock"
   branch = "master"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.8.0"

cmd/server/handler.go

@@ -103,12 +103,14 @@ func RunHost(c *config.Config) func(cmd *cobra.Command, args []string) {
 		n := negroni.New()
 
 		if ok, _ := cmd.Flags().GetBool("disable-telemetry"); !ok && os.Getenv("DISABLE_TELEMETRY") != "1" {
-			metrics := c.GetMetrics()
-			go metrics.RegisterSegment()
-			go metrics.CommitMemoryStatistics()
-			n.Use(metrics)
+			telemetryMetrics := c.GetTelemetryMetrics()
+			go telemetryMetrics.RegisterSegment()
+			go telemetryMetrics.CommitMemoryStatistics()
+			n.Use(telemetryMetrics)
 		}
 
+		n.Use(c.GetPrometheusMetrics())
+
 		n.Use(negronilogrus.NewMiddlewareFromLogger(logger, c.Issuer))
 		n.UseFunc(serverHandler.rejectInsecureRequests)
 		n.UseHandler(router)

cmd/server/handler_health_factory.go

@@ -23,7 +23,7 @@ import (
 
 func newHealthHandler(c *config.Config, router *httprouter.Router) *health.Handler {
 	h := &health.Handler{
-		Metrics:        c.GetMetrics(),
+		Metrics:        c.GetTelemetryMetrics(),
 		H:              herodot.NewJSONWriter(c.GetLogger()),
 		W:              c.Context().Warden,
 		ResourcePrefix: c.AccessControlResourcePrefix,

config/config.go

@@ -31,7 +31,8 @@ import (
 	foauth2 "github.com/ory/fosite/handler/oauth2"
 	"github.com/ory/fosite/token/hmac"
 	"github.com/ory/hydra/health"
-	"github.com/ory/hydra/metrics"
+	"github.com/ory/hydra/metrics/prometheus"
+	"github.com/ory/hydra/metrics/telemetry"
 	"github.com/ory/hydra/pkg"
 	"github.com/ory/hydra/warden/group"
 	"github.com/ory/ladon"
@@ -78,15 +79,16 @@ type Config struct {
 	ForceHTTP                        bool   `yaml:"-"`
 	JWTParseTimeWindow               uint   `mapstructure:"JWT_PARSE_TIME_WINDOW" yaml:"-"`
 
-	BuildVersion string                  `yaml:"-"`
-	BuildHash    string                  `yaml:"-"`
-	BuildTime    string                  `yaml:"-"`
-	logger       *logrus.Logger          `yaml:"-"`
-	metrics      *metrics.MetricsManager `yaml:"-"`
-	cluster      *url.URL                `yaml:"-"`
-	oauth2Client *http.Client            `yaml:"-"`
-	context      *Context                `yaml:"-"`
-	systemSecret []byte                  `yaml:"-"`
+	BuildVersion string                     `yaml:"-"`
+	BuildHash    string                     `yaml:"-"`
+	BuildTime    string                     `yaml:"-"`
+	logger       *logrus.Logger             `yaml:"-"`
+	telemetry    *telemetry.MetricsManager  `yaml:"-"`
+	prometheus   *prometheus.MetricsManager `yaml:"-"`
+	cluster      *url.URL                   `yaml:"-"`
+	oauth2Client *http.Client               `yaml:"-"`
+	context      *Context                   `yaml:"-"`
+	systemSecret []byte                     `yaml:"-"`
 }
 
 func (c *Config) GetClusterURLWithoutTailingSlash() string {
@@ -148,12 +150,21 @@ func (c *Config) GetLogger() *logrus.Logger {
 	return c.logger
 }
 
-func (c *Config) GetMetrics() *metrics.MetricsManager {
-	if c.metrics == nil {
-		c.metrics = metrics.NewMetricsManager(c.Issuer, c.DatabaseURL, c.GetLogger(), c.BuildVersion, c.BuildHash, c.BuildTime)
+func (c *Config) GetTelemetryMetrics() *telemetry.MetricsManager {
+	if c.telemetry == nil {
+		c.telemetry = telemetry.NewMetricsManager(c.Issuer, c.DatabaseURL, c.GetLogger(), c.BuildVersion, c.BuildHash, c.BuildTime)
 	}
 
-	return c.metrics
+	return c.telemetry
+}
+
+func (c *Config) GetPrometheusMetrics() *prometheus.MetricsManager {
+	if c.prometheus == nil {
+		c.GetLogger().Info("Setting up Prometheus metrics")
+		c.prometheus = prometheus.NewMetricsManager(c.BuildVersion, c.BuildHash, c.BuildTime)
+	}
+
+	return c.prometheus
 }
 
 func (c *Config) DoesRequestSatisfyTermination(r *http.Request) error {

health/handler.go

@@ -20,15 +20,17 @@ import (
 	"github.com/julienschmidt/httprouter"
 	"github.com/ory/herodot"
 	"github.com/ory/hydra/firewall"
-	"github.com/ory/hydra/metrics"
+	"github.com/ory/hydra/metrics/telemetry"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
-	HealthStatusPath = "/health/status"
+	HealthStatusPath     = "/health/status"
+	PrometheusStatusPath = "/health/prometheus"
 )
 
 type Handler struct {
-	Metrics        *metrics.MetricsManager
+	Metrics        *telemetry.MetricsManager
 	H              *herodot.JSONWriter
 	W              firewall.Firewall
 	ResourcePrefix string
@@ -48,6 +50,7 @@ func (h *Handler) PrefixResource(resource string) string {
 
 func (h *Handler) SetRoutes(r *httprouter.Router) {
 	r.GET(HealthStatusPath, h.Health)
+	r.Handler("GET", PrometheusStatusPath, promhttp.Handler())
 }
 
 // swagger:route GET /health/status health getInstanceStatus

metrics/prometheus/metrics.go

@@ -0,0 +1,73 @@
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics prototypes
+// Example:
+// 	Counter      *prometheus.CounterVec
+// 	ResponseTime *prometheus.HistogramVec
+type Metrics struct {
+	Counter      *prometheus.CounterVec
+	ResponseTime *prometheus.HistogramVec
+}
+
+// Method for creation new custom Prometheus  metrics
+// Example:
+// 	pm := &Metrics{
+//		Counter: prometheus.NewCounterVec(
+//			prometheus.CounterOpts{
+//				Name:        "servicename_requests_total",
+//				Help:        "Description",
+//				ConstLabels: map[string]string{
+//					"version":   version,
+//					"hash":      hash,
+//					"buildTime": buildTime,
+//				},
+//			},
+//			[]string{"endpoint"},
+//		),
+//		ResponseTime: prometheus.NewHistogramVec(
+//			prometheus.HistogramOpts{
+//				Name:        "servicename_response_time_seconds",
+//				Help:        "Description",
+//				ConstLabels: map[string]string{
+//					"version":   version,
+//					"hash":      hash,
+//					"buildTime": buildTime,
+//				},
+//			},
+//			[]string{"endpoint"},
+//		),
+//	}
+//	prometheus.Register(pm.Counter)
+//  prometheus.Register(pm.ResponseTime)
+func NewMetrics(version, hash, buildTime string) *Metrics {
+	labels := map[string]string{
+		"version":   version,
+		"hash":      hash,
+		"buildTime": buildTime,
+	}
+	pm := &Metrics{
+		Counter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name:        "sts_requests_total",
+				Help:        "Secure token service requests served per endpoint",
+				ConstLabels: labels,
+			},
+			[]string{"endpoint"},
+		),
+		ResponseTime: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name:        "sts_response_time_seconds",
+				Help:        "Secure token service response time",
+				ConstLabels: labels,
+			},
+			[]string{"endpoint"},
+		),
+	}
+	prometheus.Register(pm.Counter)
+	prometheus.Register(pm.ResponseTime)
+	return pm
+}

metrics/prometheus/middleware.go

@@ -0,0 +1,33 @@
+package prometheus
+
+import (
+	"net/http"
+	"time"
+)
+
+type MetricsManager struct {
+	prometheusMetrics *Metrics
+}
+
+func NewMetricsManager(version, hash, buildTime string) *MetricsManager {
+	return &MetricsManager{
+		prometheusMetrics: NewMetrics(version, hash, buildTime),
+	}
+}
+
+// Main middleware method to collect metrics for Prometheus.
+// Example:
+//	start := time.Now()
+//	next(rw, r)
+//	Request counter metric
+//	pmm.prometheusMetrics.Counter.WithLabelValues(r.URL.Path).Inc()
+//	Response time metric
+//	pmm.prometheusMetrics.ResponseTime.WithLabelValues(r.URL.Path).Observe(time.Since(start).Seconds())
+func (pmm *MetricsManager) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
+	start := time.Now()
+	next(rw, r)
+	// Request counter metric
+	pmm.prometheusMetrics.Counter.WithLabelValues(r.URL.Path).Inc()
+	// Response time metric
+	pmm.prometheusMetrics.ResponseTime.WithLabelValues(r.URL.Path).Observe(time.Since(start).Seconds())
+}

metrics/metrics.go → metrics/telemetry/metrics.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package metrics
+package telemetry
 
 import (
 	"runtime"

metrics/middleware.go → metrics/telemetry/middleware.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package metrics
+package telemetry
 
 import (
 	"crypto/sha256"
@@ -94,6 +94,7 @@ func NewMetricsManager(issuerURL string, databaseURL string, l logrus.FieldLogge
 		salt:         uuid.New(),
 		BuildTime:    buildTime, BuildVersion: version, BuildHash: hash,
 	}
+
 	return mm
 }
 

metrics/middleware_test.go → metrics/telemetry/middleware_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package metrics
+package telemetry
 
 import (
 	"net/url"