From a06c162aa79d8acdafff5609a1c30328457dae04 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Wed, 25 Mar 2020 10:57:17 -0400 Subject: [PATCH] metrics: split bytes flushed and bytes compacted metrics Previously Pebble exposed a BytesWritten metric that included both flush and compaction writes. This change replaces that metric with two distinct metcis, BytesFlushed and BytesCompacted. --- compaction.go | 3 ++- metrics.go | 24 +++++++++++++++--------- metrics_test.go | 27 ++++++++++++++------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/compaction.go b/compaction.go index 1d735fee0c..bcf2caa98e 100644 --- a/compaction.go +++ b/compaction.go @@ -1352,11 +1352,12 @@ func (d *DB) runCompaction( meta.SmallestSeqNum = writerMeta.SmallestSeqNum meta.LargestSeqNum = writerMeta.LargestSeqNum - metrics.BytesWritten += meta.Size if c.flushing == nil { metrics.TablesCompacted++ + metrics.BytesCompacted += meta.Size } else { metrics.TablesFlushed++ + metrics.BytesFlushed += meta.Size } // The handling of range boundaries is a bit complicated. diff --git a/metrics.go b/metrics.go index a885b535ed..7ab0ad6a39 100644 --- a/metrics.go +++ b/metrics.go @@ -49,9 +49,14 @@ type LevelMetrics struct { // The number of bytes read for compactions at the level. This includes bytes // read from other levels (BytesIn), as well as bytes read for the level. BytesRead uint64 - // The number of bytes written during flushes and compactions. The sibling - // metrics for tables are TablesCompacted and TablesFlushed. - BytesWritten uint64 + // The number of bytes written during compactions. The sibling + // metric for tables is TablesCompacted. This metric may be summed + // with BytesFlushed to compute the total bytes written for the level. + BytesCompacted uint64 + // The number of bytes written during flushes. The sibling + // metrics for tables is TablesFlushed. This metric is always + // zero for all levels other than L0. + BytesFlushed uint64 // The number of sstables compacted to this level. TablesCompacted uint64 // The number of sstables flushed to this level. @@ -68,7 +73,8 @@ func (m *LevelMetrics) Add(u *LevelMetrics) { m.BytesIngested += u.BytesIngested m.BytesMoved += u.BytesMoved m.BytesRead += u.BytesRead - m.BytesWritten += u.BytesWritten + m.BytesCompacted += u.BytesCompacted + m.BytesFlushed += u.BytesFlushed m.TablesCompacted += u.TablesCompacted m.TablesFlushed += u.TablesFlushed m.TablesIngested += u.TablesIngested @@ -76,12 +82,12 @@ func (m *LevelMetrics) Add(u *LevelMetrics) { } // WriteAmp computes the write amplification for compactions at this -// level. Computed as BytesWritten / BytesIn. +// level. Computed as (BytesFlushed + BytesCompacted) / BytesIn. func (m *LevelMetrics) WriteAmp() float64 { if m.BytesIn == 0 { return 0 } - return float64(m.BytesWritten) / float64(m.BytesIn) + return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn) } // format generates a string of the receiver's metrics, formatting it into the @@ -96,7 +102,7 @@ func (m *LevelMetrics) format(buf *bytes.Buffer, score string) { humanize.SI.Uint64(m.TablesIngested), humanize.IEC.Uint64(m.BytesMoved), humanize.SI.Uint64(m.TablesMoved), - humanize.IEC.Uint64(m.BytesWritten), + humanize.IEC.Uint64(m.BytesFlushed+m.BytesCompacted), humanize.SI.Uint64(m.TablesFlushed+m.TablesCompacted), humanize.IEC.Uint64(m.BytesRead), m.WriteAmp()) @@ -235,10 +241,10 @@ func (m *Metrics) String() string { } // Compute total bytes-in as the bytes written to the WAL + bytes ingested total.BytesIn = m.WAL.BytesWritten + total.BytesIngested - // Add the total bytes-in to the total bytes-written. This is to account for + // Add the total bytes-in to the total bytes-flushed. This is to account for // the bytes written to the log and bytes written externally and then // ingested. - total.BytesWritten += total.BytesIn + total.BytesFlushed += total.BytesIn fmt.Fprintf(&buf, " total ") total.format(&buf, "-") diff --git a/metrics_test.go b/metrics_test.go index e4beb3e66d..e2cad75534 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -52,24 +52,25 @@ func TestMetricsFormat(t *testing.T) { l.BytesIngested = base + 4 l.BytesMoved = base + 6 l.BytesRead = base + 7 - l.BytesWritten = base + 8 - l.TablesCompacted = base + 9 - l.TablesFlushed = base + 10 - l.TablesIngested = base + 11 - l.TablesMoved = base + 12 + l.BytesCompacted = base + 8 + l.BytesFlushed = base + 9 + l.TablesCompacted = base + 10 + l.TablesFlushed = base + 11 + l.TablesIngested = base + 12 + l.TablesMoved = base + 13 } const expected = ` __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___w-amp WAL 21 23 B - 24 B - - - - 25 B - - 1.0 - 0 101 102 B 103.00 104 B 104 B 111 106 B 112 108 B 219 107 B 1.0 - 1 201 202 B 203.00 204 B 204 B 211 206 B 212 208 B 419 207 B 1.0 - 2 301 302 B 303.00 304 B 304 B 311 306 B 312 308 B 619 307 B 1.0 - 3 401 402 B 403.00 404 B 404 B 411 406 B 412 408 B 819 407 B 1.0 - 4 501 502 B 503.00 504 B 504 B 511 506 B 512 508 B 1.0 K 507 B 1.0 - 5 601 602 B 603.00 604 B 604 B 611 606 B 612 608 B 1.2 K 607 B 1.0 - 6 701 702 B 703.00 704 B 704 B 711 706 B 712 708 B 1.4 K 707 B 1.0 - total 2807 2.7 K - 2.8 K 2.8 K 2.9 K 2.8 K 2.9 K 5.6 K 5.7 K 2.8 K 2.0 + 0 101 102 B 103.00 104 B 104 B 112 106 B 113 217 B 221 107 B 2.1 + 1 201 202 B 203.00 204 B 204 B 212 206 B 213 417 B 421 207 B 2.0 + 2 301 302 B 303.00 304 B 304 B 312 306 B 313 617 B 621 307 B 2.0 + 3 401 402 B 403.00 404 B 404 B 412 406 B 413 817 B 821 407 B 2.0 + 4 501 502 B 503.00 504 B 504 B 512 506 B 513 1017 B 1.0 K 507 B 2.0 + 5 601 602 B 603.00 604 B 604 B 612 606 B 613 1.2 K 1.2 K 607 B 2.0 + 6 701 702 B 703.00 704 B 704 B 712 706 B 713 1.4 K 1.4 K 707 B 2.0 + total 2807 2.7 K - 2.8 K 2.8 K 2.9 K 2.8 K 2.9 K 8.4 K 5.7 K 2.8 K 3.0 flush 7 compact 5 6 B (size == estimated-debt) memtbl 11 10 B