feature: pouch metrics

Signed-off-by: codejuan <xh@decbug.com>

apis/metrics/metrics.go

@@ -1,6 +1,7 @@
 package metrics
 
 import (
+	"fmt"
 	"sync"
 	"time"
 
@@ -13,19 +14,32 @@ func init() {
 }
 
 const (
-	pouchSubsystem = "pouch"
+	namespace = "engine"
+	subsystem = "daemon"
 )
 
 var (
 	// ImagePullSummary records the summary of pulling image latency.
 	ImagePullSummary = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
-			Subsystem: pouchSubsystem,
+			Namespace: namespace,
+			Subsystem: subsystem,
 			Name:      "image_pull_latency_microseconds",
 			Help:      "Latency in microseconds to pull a image.",
 		},
 		[]string{"image"},
 	)
+
+	ContainerActionsCounter        = newLabelCounter("container_actions_counter", "The number of container operations", "action")
+	ContainerSuccessActionsCounter = newLabelCounter("container_success_actions_counter", "The number of container success operations", "action")
+	ImageActionsCounter            = newLabelCounter("image_actions_counter", "The number of image operations", "action")
+	ImageSuccessActionsCounter     = newLabelCounter("image_success_actions_counter", "The number of image success operations", "action")
+	ContainerActionsTimer          = newLabelTimer("container_actions", "The number of seconds it takes to process each container action", "action")
+	ImageActionsTimer              = newLabelTimer("image_actions", "The number of seconds it takes to process each image action", "action")
+
+	EngineVersion = newLabelGauge("engine", "The version and commit information for the engine process",
+		"commit",
+	)
 )
 
 var registerMetrics sync.Once
@@ -35,10 +49,51 @@ func Register() {
 	// Register the metrics.
 	registerMetrics.Do(func() {
 		prometheus.MustRegister(ImagePullSummary)
+		prometheus.MustRegister(EngineVersion)
+		prometheus.MustRegister(ContainerActionsCounter)
+		prometheus.MustRegister(ContainerSuccessActionsCounter)
+		prometheus.MustRegister(ImageActionsCounter)
+		prometheus.MustRegister(ImageSuccessActionsCounter)
+		prometheus.MustRegister(ContainerActionsTimer)
+		prometheus.MustRegister(ImageActionsTimer)
 	})
 }
 
 // SinceInMicroseconds gets the time since the specified start in microseconds.
 func SinceInMicroseconds(start time.Time) float64 {
 	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
 }
+
+func newLabelCounter(name, help string, labels ...string) *prometheus.CounterVec {
+	return prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   namespace,
+			Subsystem:   subsystem,
+			Name:        fmt.Sprintf("%s_%s", name, Total),
+			Help:        help,
+			ConstLabels: nil,
+		},
+		labels)
+}
+
+func newLabelGauge(name, help string, labels ...string) *prometheus.GaugeVec {
+	return prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace:   namespace,
+			Subsystem:   subsystem,
+			Name:        fmt.Sprintf("%s_%s", name, Unit("info")),
+			Help:        help,
+			ConstLabels: nil,
+		}, labels)
+}
+
+func newLabelTimer(name, help string, labels ...string) *prometheus.HistogramVec {
+	return prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace:   namespace,
+			Subsystem:   subsystem,
+			Name:        fmt.Sprintf("%s_%s", name, Seconds),
+			Help:        help,
+			ConstLabels: nil,
+		}, labels)
+}

apis/metrics/unit.go

@@ -0,0 +1,12 @@
+package metrics
+
+// Unit represents the type or precision of a metric that is appended to
+// the metrics fully qualified name
+type Unit string
+
+const (
+	Nanoseconds Unit = "nanoseconds"
+	Seconds     Unit = "seconds"
+	Bytes       Unit = "bytes"
+	Total       Unit = "total"
+)

apis/server/container_bridge.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/alibaba/pouch/apis/metrics"
 	"github.com/alibaba/pouch/apis/types"
 	"github.com/alibaba/pouch/daemon/mgr"
 	"github.com/alibaba/pouch/pkg/httputils"
@@ -22,6 +23,12 @@ import (
 )
 
 func (s *Server) createContainer(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
+	label := "create"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	config := &types.ContainerCreateConfig{}
 	reader := req.Body
 	var ex error
@@ -57,6 +64,8 @@ func (s *Server) createContainer(ctx context.Context, rw http.ResponseWriter, re
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	return EncodeResponse(rw, http.StatusCreated, container)
 }
 
@@ -150,6 +159,12 @@ func (s *Server) getContainers(ctx context.Context, rw http.ResponseWriter, req
 }
 
 func (s *Server) startContainer(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
+	label := "start"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	name := mux.Vars(req)["name"]
 
 	options := &types.ContainerStartOptions{
@@ -162,6 +177,8 @@ func (s *Server) startContainer(ctx context.Context, rw http.ResponseWriter, req
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }
@@ -171,6 +188,11 @@ func (s *Server) restartContainer(ctx context.Context, rw http.ResponseWriter, r
 		t   int
 		err error
 	)
+	label := "restart"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
 
 	if v := req.FormValue("t"); v != "" {
 		if t, err = strconv.Atoi(v); err != nil {
@@ -184,6 +206,8 @@ func (s *Server) restartContainer(ctx context.Context, rw http.ResponseWriter, r
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }
@@ -194,6 +218,12 @@ func (s *Server) stopContainer(ctx context.Context, rw http.ResponseWriter, req
 		err error
 	)
 
+	label := "stop"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	if v := req.FormValue("t"); v != "" {
 		if t, err = strconv.Atoi(v); err != nil {
 			return httputils.NewHTTPError(err, http.StatusBadRequest)
@@ -206,6 +236,8 @@ func (s *Server) stopContainer(ctx context.Context, rw http.ResponseWriter, req
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }
@@ -233,13 +265,21 @@ func (s *Server) unpauseContainer(ctx context.Context, rw http.ResponseWriter, r
 }
 
 func (s *Server) renameContainer(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
+	label := "rename"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	oldName := mux.Vars(req)["name"]
 	newName := req.FormValue("name")
 
 	if err := s.ContainerMgr.Rename(ctx, oldName, newName); err != nil {
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }
@@ -270,6 +310,12 @@ func (s *Server) attachContainer(ctx context.Context, rw http.ResponseWriter, re
 }
 
 func (s *Server) updateContainer(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
+	label := "update"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	config := &types.UpdateConfig{}
 
 	// set pre update hook plugin
@@ -293,11 +339,19 @@ func (s *Server) updateContainer(ctx context.Context, rw http.ResponseWriter, re
 		return httputils.NewHTTPError(err, http.StatusInternalServerError)
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusOK)
 	return nil
 }
 
 func (s *Server) upgradeContainer(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
+	label := "upgrade"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	config := &types.ContainerUpgradeConfig{}
 	// decode request body
 	if err := json.NewDecoder(req.Body).Decode(config); err != nil {
@@ -314,6 +368,8 @@ func (s *Server) upgradeContainer(ctx context.Context, rw http.ResponseWriter, r
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusOK)
 	return nil
 }
@@ -399,6 +455,12 @@ func (s *Server) resizeContainer(ctx context.Context, rw http.ResponseWriter, re
 }
 
 func (s *Server) removeContainers(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
+	label := "delete"
+	metrics.ContainerActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ContainerActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	name := mux.Vars(req)["name"]
 
 	option := &types.ContainerRemoveOptions{
@@ -412,6 +474,8 @@ func (s *Server) removeContainers(ctx context.Context, rw http.ResponseWriter, r
 		return err
 	}
 
+	metrics.ContainerSuccessActionsCounter.WithLabelValues(label).Inc()
+
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }

apis/server/image_bridge.go

@@ -34,9 +34,13 @@ func (s *Server) pullImage(ctx context.Context, rw http.ResponseWriter, req *htt
 		image = image + ":" + tag
 	}
 
+	label := "pull"
+	metrics.ImageActionsCounter.WithLabelValues(label).Inc()
+
 	// record the time spent during image pull procedure.
 	defer func(start time.Time) {
 		metrics.ImagePullSummary.WithLabelValues(image).Observe(metrics.SinceInMicroseconds(start))
+		metrics.ImageActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
 	}(time.Now())
 
 	// get registry auth from Request header
@@ -53,6 +57,7 @@ func (s *Server) pullImage(ctx context.Context, rw http.ResponseWriter, req *htt
 		logrus.Errorf("failed to pull image %s: %v", image, err)
 		return nil
 	}
+	metrics.ImageSuccessActionsCounter.WithLabelValues(label).Inc()
 	return nil
 }
 
@@ -105,6 +110,12 @@ func (s *Server) removeImage(ctx context.Context, rw http.ResponseWriter, req *h
 		return err
 	}
 
+	label := "delete"
+	metrics.ImageActionsCounter.WithLabelValues(label).Inc()
+	defer func(start time.Time) {
+		metrics.ImageActionsTimer.WithLabelValues(label).Observe(time.Since(start).Seconds())
+	}(time.Now())
+
 	isForce := httputils.BoolValue(req, "force")
 	// We only should check the image whether used by container when there is only one primary reference.
 	if len(refs) == 1 {
@@ -126,6 +137,7 @@ func (s *Server) removeImage(ctx context.Context, rw http.ResponseWriter, req *h
 		return err
 	}
 
+	metrics.ImageSuccessActionsCounter.WithLabelValues(label).Inc()
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }

main.go

@@ -9,6 +9,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/alibaba/pouch/apis/metrics"
 	"github.com/alibaba/pouch/apis/opts"
 	optscfg "github.com/alibaba/pouch/apis/opts/config"
 	"github.com/alibaba/pouch/apis/types"
@@ -150,7 +151,7 @@ func runDaemon(cmd *cobra.Command) error {
 		fmt.Printf("pouchd version: %s, build: %s, build at: %s\n", version.Version, version.GitCommit, version.BuildTime)
 		return nil
 	}
-
+	metrics.EngineVersion.WithLabelValues(version.GitCommit).Set(1)
 	// initialize log.
 	initLog()