metrics: improve metrics

docs/telemetry.md

@@ -73,6 +73,7 @@ Additionally, the following data is submitted to us:
 * Total number of requests, responses, response latencies and response sizes per HTTP method.
 * Total number of requests, responses, response latencies and response sizes per anonymized API endpoint and HTTP method.
 * Total number of requests, responses, response latencies and response sizes per anonymized API endpoint.
+* Memory statistics such as heap allocations, gc cycles, and other.
 
 A raw data example can be found [here](https://github.com/ory/hydra/tree/master/docs/metrics/telemetry-example.json).
 

health/handler.go

@@ -25,7 +25,7 @@ func (h *Handler) SetRoutes(r *httprouter.Router) {
 //       204: emptyResponse
 //       500: genericError
 func (h *Handler) Health(rw http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-	h.Metrics.UpdateUpTime()
+	h.Metrics.Update()
 
 	h.Metrics.RLock()
 	defer h.Metrics.RUnlock()

metrics/metrics.go

@@ -1,6 +1,10 @@
 package metrics
 
-import "time"
+import (
+	"time"
+	"sync"
+	"runtime"
+)
 
 type Metrics struct {
 	Requests  uint64        `json:"requests,omitempty"`
@@ -31,12 +35,7 @@ func (h *HTTPMetrics) AddMethodRequest(method string) {
 }
 
 func (h *Metrics) AddLatency(latency time.Duration) {
-	if latency > time.Second*5 {
-		latency = time.Second * 5
-	}
-
-	// milliseconds / 10
-	h.Latencies[int64(latency/10)]++
+	h.Latencies[int64(latency)]++
 }
 
 func (h *HTTPMetrics) SizeMetrics(size int) *Metrics {
@@ -89,9 +88,30 @@ type PathMetrics struct {
 }
 
 type Snapshot struct {
+	sync.RWMutex
 	*Metrics
 	*HTTPMetrics
 	Paths map[string]*PathMetrics `json:"paths"`
+	ID           string `json:"id"`
+	UpTime       int64  `json:"uptime"`
+	start        time.Time          `json:"-"`
+	MemorySnapshot *MemorySnapshot `json:"memory"`
+}
+
+type MemorySnapshot struct {
+	Alloc uint64 `json:"alloc"`
+	TotalAlloc uint64 `json:"totalAlloc"`
+	Sys uint64 `json:"sys"`
+	Lookups uint64 `json:"lookups"`
+	Mallocs uint64 `json:"mallocs"`
+	Frees uint64 `json:"frees"`
+	HeapAlloc uint64 `json:"heapAlloc"`
+	HeapSys uint64 `json:"heapSys"`
+	HeapIdle uint64 `json:"heapIdle"`
+	HeapInuse uint64 `json:"heapInuse"`
+	HeapReleased uint64 `json:"heapReleased"`
+	HeapObjects uint64 `json:"heapObjects"`
+	NumGC uint32 `json:"numGC"`
 }
 
 func newMetrics() *Metrics {
@@ -100,6 +120,40 @@ func newMetrics() *Metrics {
 	}
 }
 
+func (sw *Snapshot) GetUpTime() int64 {
+	sw.Update()
+	sw.RLock()
+	defer sw.RUnlock()
+	return sw.UpTime
+}
+
+func (sw *Snapshot) Update()  {
+	sw.Lock()
+	defer sw.Unlock()
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	// sw.MemorySnapshot = &(MemorySnapshot(m))
+	sw.MemorySnapshot = &MemorySnapshot{
+		Alloc: m.Alloc,
+		TotalAlloc: m.TotalAlloc,
+		Sys: m.Sys,
+		Lookups: m.Lookups,
+		Mallocs: m.Mallocs,
+		Frees: m.Frees,
+		HeapAlloc: m.HeapAlloc,
+		HeapSys: m.HeapSys,
+		HeapIdle: m.HeapIdle,
+		HeapInuse: m.HeapInuse,
+		HeapReleased: m.HeapReleased,
+		HeapObjects: m.HeapObjects,
+		NumGC: m.NumGC,
+	}
+	sw.UpTime = int64(time.Now().Sub(sw.start) / time.Second)
+
+}
+
 func (s *Snapshot) Path(path string) *PathMetrics {
 	paths := []string{
 		"/.well-known/jwks.json",

metrics/middleware.go

@@ -16,12 +16,9 @@ import (
 
 type MetricsManager struct {
 	sync.RWMutex `json:"-"`
-	ID           string `json:"id"`
-	UpTime       int64  `json:"uptime"`
 	*Snapshot
 	Segment      *analytics.Client  `json:"-"`
 	Logger       logrus.FieldLogger `json:"-"`
-	start        time.Time          `json:"-"`
 	buildVersion string
 	buildHash    string
 	buildTime    string
@@ -31,25 +28,19 @@ func NewMetricsManager(l logrus.FieldLogger) *MetricsManager {
 	l.Info("Setting up telemetry - for more information please visit https://ory.gitbooks.io/hydra/content/telemetry.html")
 	mm := &MetricsManager{
 		Snapshot: &Snapshot{
-			Metrics:     newMetrics(),
-			HTTPMetrics: newHttpMetrics(),
-			Paths:       map[string]*PathMetrics{},
+			MemorySnapshot: &MemorySnapshot{},
+			ID:             uuid.New(),
+			Metrics:        newMetrics(),
+			HTTPMetrics:    newHttpMetrics(),
+			Paths:          map[string]*PathMetrics{},
+			start:          time.Now(),
 		},
-		ID:      uuid.New(),
 		Segment: analytics.New("JYilhx5zP8wrzfykUinXrSUbo5cRA3aA"),
 		Logger:  l,
-		start:   time.Now(),
 	}
 	return mm
 }
 
-func (sw *MetricsManager) UpdateUpTime() {
-	sw.Lock()
-	defer sw.Unlock()
-	sw.UpTime = int64(time.Now().Sub(sw.start) / time.Second)
-
-}
-
 const (
 	defaultWait   = time.Minute * 15
 	keepAliveWait = time.Minute * 5
@@ -98,6 +89,7 @@ func (sw *MetricsManager) TickKeepAlive() {
 func (sw *MetricsManager) CommitTelemetry() {
 	for {
 		time.Sleep(defaultWait)
+		sw.Update()
 		if err := sw.Segment.Track(&analytics.Track{
 			Event:       "telemetry",
 			AnonymousId: sw.ID,
@@ -111,6 +103,7 @@ func (sw *MetricsManager) CommitTelemetry() {
 				"status":    sw.Status,
 				"latencies": sw.Latencies,
 				"raw":       sw,
+				"memory":    sw.MemorySnapshot,
 			},
 			Context: map[string]interface{}{
 				"ip": "0.0.0.0",

metrics/middleware_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/ory/herodot"
 	"github.com/ory/hydra/health"
+	"encoding/json"
 )
 
 func TestMiddleware(t *testing.T) {
@@ -54,6 +55,10 @@ func TestMiddleware(t *testing.T) {
 	assert.EqualValues(t, i, mw.Snapshot.Requests)
 	assert.EqualValues(t, i, mw.Snapshot.Responses)
 
+	mw.Snapshot.Update()
+	assert.True(t, mw.Snapshot.UpTime > 0)
+	assert.True(t, mw.Snapshot.GetUpTime() > 0)
+
 	assert.EqualValues(t, 0, mw.Snapshot.Status[http.StatusOK].Requests)
 	assert.EqualValues(t, i, mw.Snapshot.Status[http.StatusOK].Responses)
 
@@ -66,8 +71,11 @@ func TestMiddleware(t *testing.T) {
 
 	assert.EqualValues(t, 1, mw.Snapshot.Path("/oauth2/introspect").Requests)
 
-	mw.UpdateUpTime()
+	mw.Update()
 	assert.NotEqual(t, 0, mw.UpTime)
+
+	out, _ := json.MarshalIndent(mw, "\t", "  ")
+	t.Logf("%s", out)
 }
 
 func TestRacyMiddleware(t *testing.T) {