metrics: collect the metrics of nydusd events

Collect the metrics of nydus daemon events, including INIT, RUNNING and DIED.

Signed-off-by: Bin Tang <tangbin.bin@bytedance.com>

pkg/daemon/daemon.go

@@ -20,6 +20,7 @@ import (
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter"
 	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/erofs"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/mount"
@@ -188,6 +189,10 @@ func (d *Daemon) WaitUntilState(expected types.DaemonState) error {
 					d.ID(), expected, state)
 			}
 
+			if exportErr := exporter.RecordDaemonEvent(d.ID(), string(expected)); exportErr != nil {
+				log.L.Warnf("export nydusd event metric failed, daemon ID: %s, event: %s, error: %v", d.States.ID, string(expected), exportErr)
+			}
+
 			return nil
 		},
 			retry.Attempts(20), // totally wait for 2 seconds, should be enough

pkg/daemon/types/types.go

@@ -24,10 +24,12 @@ type DaemonInfo struct {
 }
 
 const (
-	DaemonStateUnknown DaemonState = "UNKNOWN"
-	DaemonStateInit    DaemonState = "INIT"
-	DaemonStateReady   DaemonState = "READY"
-	DaemonStateRunning DaemonState = "RUNNING"
+	DaemonStateUnknown   DaemonState = "UNKNOWN"
+	DaemonStateInit      DaemonState = "INIT"
+	DaemonStateReady     DaemonState = "READY"
+	DaemonStateRunning   DaemonState = "RUNNING"
+	DaemonStateDied      DaemonState = "DIED"
+	DaemonStateDestroyed DaemonState = "DESTROYED"
 )
 
 func (info *DaemonInfo) DaemonState() DaemonState {

pkg/manager/manager.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containerd/nydus-snapshotter/pkg/daemon"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter"
 	"github.com/containerd/nydus-snapshotter/pkg/store"
 	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
 )
@@ -506,6 +507,10 @@ func (m *Manager) DestroyDaemon(d *daemon.Daemon) error {
 		log.L.Warnf("Failed to wait for daemon, %v", err)
 	}
 
+	if err := exporter.RecordDaemonEvent(d.ID(), string(types.DaemonStateDestroyed)); err != nil {
+		log.L.Warnf("export nydusd event metric failed, daemon ID: %s, event: %s, error: %v", d.ID(), string(types.DaemonStateDestroyed), err)
+	}
+
 	return nil
 }
 

pkg/manager/monitor.go

@@ -16,7 +16,9 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/retry"
 )
 
@@ -217,6 +219,9 @@ func (m *livenessMonitor) Run() {
 
 				if ev.Events&(unix.EPOLLHUP|unix.EPOLLERR) != 0 {
 					log.L.Warnf("Daemon %s died", target.id)
+					if err := exporter.RecordDaemonEvent(target.id, string(types.DaemonStateDied)); err != nil {
+						log.L.Warnf("export nydusd event metric failed, daemon ID: %s, event: %s, error: %v", target.id, string(types.DaemonStateDied), err)
+					}
 					// Notify subscribers that death event happens
 					target.notifier <- deathEvent{daemonID: target.id, path: target.path}
 				}

pkg/metrics/exporter/daemon/metrics.go

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemon
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	daemonIDLabel = "daemon_id"
+	timeLabel     = "time"
+	eventLabel    = "event"
+)
+
+var (
+	NydusdEvent = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "nydusd_lifetime_events",
+			Help: "The lifetime events of nydus daemon.",
+		},
+		[]string{daemonIDLabel, timeLabel, eventLabel},
+	)
+)

pkg/metrics/exporter/export.go

@@ -16,8 +16,6 @@ import (
 	"github.com/pkg/errors"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
-
-	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 )
 
 type Opt func(*Exporter) error
@@ -26,6 +24,8 @@ type Exporter struct {
 	outputFile string
 }
 
+var globalExporter *Exporter
+
 func WithOutputFile(metricsFile string) Opt {
 	return func(e *Exporter) error {
 		if metricsFile == "" {
@@ -40,32 +40,33 @@ func WithOutputFile(metricsFile string) Opt {
 	}
 }
 
-func NewExporter(opts ...Opt) (*Exporter, error) {
+func NewExporter(opts ...Opt) error {
 	var exp Exporter
 
 	for _, o := range opts {
 		if err := o(&exp); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
-	return &exp, nil
-}
+	globalExporter = &exp
 
-func (e *Exporter) ExportFsMetrics(m *types.FsMetrics, imageRef string) error {
-	ReadCount.WithLabelValues(imageRef).Set(float64(m.DataRead))
-	OpenFdCount.WithLabelValues(imageRef).Set(float64(m.NrOpens))
-	OpenFdMaxCount.WithLabelValues(imageRef).Set(float64(m.NrMaxOpens))
-	LastFopTimestamp.WithLabelValues(imageRef).Set(float64(m.LastFopTp))
+	return nil
+}
 
-	for _, h := range FsMetricHists {
-		o, err := h.ToConstHistogram(m, imageRef)
-		if err != nil {
-			return errors.Wrapf(err, "failed to new const histogram for %s", h.Desc.String())
-		}
-		h.Save(o)
+func getExporter() (*Exporter, error) {
+	if globalExporter == nil {
+		return nil, errors.New("failed to get global metric exporter")
 	}
 
+	return globalExporter, nil
+}
+
+func ExportOutput() error {
+	e, err := getExporter()
+	if err != nil {
+		return err
+	}
 	return e.output()
 }
 

pkg/metrics/exporter/metrics.go → pkg/metrics/exporter/fs/metrics.go

@@ -4,11 +4,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package exporter
+package fs
 
 import (
-	"time"
-
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
@@ -17,7 +15,6 @@ import (
 
 var (
 	imageRefLabel = "image_ref"
-	defaultTTL    = 3 * time.Minute
 )
 
 var (
@@ -28,7 +25,7 @@ var (
 			Help: "Total number read of a nydus fs, in Byte.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 
 	OpenFdCount = ttl.NewGaugeVecWithTTL(
@@ -37,7 +34,7 @@ var (
 			Help: "Number of current open files.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 
 	OpenFdMaxCount = ttl.NewGaugeVecWithTTL(
@@ -46,7 +43,7 @@ var (
 			Help: "Number of max open files.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 
 	LastFopTimestamp = ttl.NewGaugeVecWithTTL(
@@ -55,12 +52,12 @@ var (
 			Help: "Timestamp of last file operation.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 )
 
 // Fs metric histograms
-var FsMetricHists = []*FsMetricHistogram{
+var MetricHists = []*MetricHistogram{
 	{
 		Desc: prometheus.NewDesc(
 			"nydusd_block_count_read_hist",

pkg/metrics/exporter/types.go → pkg/metrics/exporter/fs/types.go

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package exporter
+package fs
 
 import (
 	"fmt"
@@ -50,7 +50,7 @@ func MakeFopBuckets() []uint64 {
 
 type GetCountersFn func(*types.FsMetrics) []uint64
 
-type FsMetricHistogram struct {
+type MetricHistogram struct {
 	Desc        *prometheus.Desc
 	Buckets     []uint64
 	GetCounters GetCountersFn
@@ -59,7 +59,7 @@ type FsMetricHistogram struct {
 	constHist prometheus.Metric
 }
 
-func (h *FsMetricHistogram) ToConstHistogram(m *types.FsMetrics, imageRef string) (prometheus.Metric, error) {
+func (h *MetricHistogram) ToConstHistogram(m *types.FsMetrics, imageRef string) (prometheus.Metric, error) {
 	var count, sum uint64
 	counters := h.GetCounters(m)
 	hmap := make(map[float64]uint64)
@@ -82,18 +82,18 @@ func (h *FsMetricHistogram) ToConstHistogram(m *types.FsMetrics, imageRef string
 	), nil
 }
 
-func (h *FsMetricHistogram) Save(m prometheus.Metric) {
+func (h *MetricHistogram) Save(m prometheus.Metric) {
 	h.constHist = m
 }
 
 // Implement prometheus.Collector interface
-func (h *FsMetricHistogram) Describe(ch chan<- *prometheus.Desc) {
+func (h *MetricHistogram) Describe(ch chan<- *prometheus.Desc) {
 	if h.Desc != nil {
 		ch <- h.Desc
 	}
 }
 
-func (h *FsMetricHistogram) Collect(ch chan<- prometheus.Metric) {
+func (h *MetricHistogram) Collect(ch chan<- prometheus.Metric) {
 	if h.constHist != nil {
 		ch <- h.constHist
 	}

pkg/metrics/exporter/record.go

@@ -0,0 +1,41 @@
+package exporter
+
+import (
+	"time"
+
+	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter/daemon"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter/fs"
+	"github.com/pkg/errors"
+)
+
+func RecordFsMetrics(m *types.FsMetrics, imageRef string) error {
+	if _, err := getExporter(); err != nil {
+		return err
+	}
+
+	fs.ReadCount.WithLabelValues(imageRef).Set(float64(m.DataRead))
+	fs.OpenFdCount.WithLabelValues(imageRef).Set(float64(m.NrOpens))
+	fs.OpenFdMaxCount.WithLabelValues(imageRef).Set(float64(m.NrMaxOpens))
+	fs.LastFopTimestamp.WithLabelValues(imageRef).Set(float64(m.LastFopTp))
+
+	for _, h := range fs.MetricHists {
+		o, err := h.ToConstHistogram(m, imageRef)
+		if err != nil {
+			return errors.Wrapf(err, "failed to new const histogram for %s", h.Desc.String())
+		}
+		h.Save(o)
+	}
+
+	return nil
+}
+
+func RecordDaemonEvent(daemonID string, event string) error {
+	if _, err := getExporter(); err != nil {
+		return err
+	}
+
+	daemon.NydusdEvent.WithLabelValues(daemonID, time.Now().Format("2006-01-02 15:04:05.000"), event).Inc()
+
+	return nil
+}

pkg/metrics/exporter/registry.go

@@ -7,6 +7,8 @@
 package exporter
 
 import (
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter/daemon"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/exporter/fs"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -16,13 +18,14 @@ var (
 
 func init() {
 	Registry.MustRegister(
-		ReadCount,
-		OpenFdCount,
-		OpenFdMaxCount,
-		LastFopTimestamp,
+		fs.ReadCount,
+		fs.OpenFdCount,
+		fs.OpenFdMaxCount,
+		fs.LastFopTimestamp,
+		daemon.NydusdEvent,
 	)
 
-	for _, m := range FsMetricHists {
+	for _, m := range fs.MetricHists {
 		Registry.MustRegister(m)
 	}
 }