diff --git a/etcdserver/api/snap/db.go b/etcdserver/api/snap/db.go index 7cb7cfb0192c..be0933541944 100644 --- a/etcdserver/api/snap/db.go +++ b/etcdserver/api/snap/db.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/coreos/etcd/pkg/fileutil" @@ -33,6 +34,8 @@ var ErrNoDBSnapshot = errors.New("snap: snapshot file doesn't exist") // SaveDBFrom saves snapshot of the database from the given reader. It // guarantees the save operation is atomic. func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { + start := time.Now() + f, err := ioutil.TempFile(s.dir, "tmp") if err != nil { return 0, err @@ -40,7 +43,9 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { var n int64 n, err = io.Copy(f, r) if err == nil { + fsyncStart := time.Now() err = fileutil.Fsync(f) + snapDBFsyncSec.Observe(time.Since(fsyncStart).Seconds()) } f.Close() if err != nil { @@ -69,6 +74,7 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { plog.Infof("saved database snapshot to disk [total bytes: %d]", n) } + snapDBSaveSec.Observe(time.Since(start).Seconds()) return n, nil } diff --git a/etcdserver/api/snap/metrics.go b/etcdserver/api/snap/metrics.go index 98003f54c442..434e81229d0d 100644 --- a/etcdserver/api/snap/metrics.go +++ b/etcdserver/api/snap/metrics.go @@ -49,10 +49,34 @@ var ( // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) + + snapDBSaveSec = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "etcd", + Subsystem: "snap_db", + Name: "save_total_duration_seconds", + Help: "The total latency distributions of v3 snapshot save", + + // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 + // highest bucket start of 0.001 sec * 2^13 == 8.192 sec + Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), + }) + + snapDBFsyncSec = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "etcd", + Subsystem: "snap_db", + Name: "fsync_duration_seconds", + Help: "The latency distributions of fsyncing .snap.db file", + + // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 + // highest bucket start of 0.001 sec * 2^13 == 8.192 sec + Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), + }) ) func init() { prometheus.MustRegister(snapMarshallingSec) prometheus.MustRegister(snapSaveSec) prometheus.MustRegister(snapFsyncSec) + prometheus.MustRegister(snapDBSaveSec) + prometheus.MustRegister(snapDBFsyncSec) }