From b5528e9dd8dd9a425680b69a220893426168d1dd Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Wed, 2 Dec 2020 18:52:31 -0800 Subject: [PATCH 1/4] feat(metric): Add dgraph_memory_alloc_bytes. --- x/metrics.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/x/metrics.go b/x/metrics.go index 78e728f4122..819d1762bb1 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -70,6 +70,9 @@ var ( // PendingProposals records the current number of pending RAFT proposals. PendingProposals = stats.Int64("pending_proposals_total", "Number of pending proposals", stats.UnitDimensionless) + // MemoryInUse records the amount of memory allocated via jemalloc/Go + MemoryAlloc = stats.Int64("memory_alloc_bytes", + "Amount of memory allocated", stats.UnitBytes) // MemoryInUse records the current amount of used memory by Dgraph. MemoryInUse = stats.Int64("memory_inuse_bytes", "Amount of memory in use", stats.UnitBytes) @@ -192,6 +195,13 @@ var ( Aggregation: view.LastValue(), TagKeys: nil, }, + { + Name: MemoryAlloc.Name(), + Measure: MemoryAlloc, + Description: MemoryAlloc.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, { Name: MemoryInUse.Name(), Measure: MemoryInUse, @@ -447,8 +457,12 @@ func MonitorMemoryMetrics(lc *z.Closer) { defer lc.Done() ticker := time.NewTicker(time.Minute) defer ticker.Stop() + fastTicker := time.NewTicker(time.Second) + defer fastTicker.Stop() update := func() { + // ReadMemStats stops the world, so don't call it too frequently. + // Calling it every minute is OK. var ms runtime.MemStats runtime.ReadMemStats(&ms) @@ -475,6 +489,8 @@ func MonitorMemoryMetrics(lc *z.Closer) { select { case <-lc.HasBeenClosed(): return + case <-fastTicker.C: + ostats.Record(context.Background(), MemoryAlloc.M(z.NumAllocBytes())) case <-ticker.C: update() } From 4403f6e55d4c8196e0b45d153a04b342c13fa5bd Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Wed, 2 Dec 2020 19:37:21 -0800 Subject: [PATCH 2/4] Update ReadMemStats comment. --- x/metrics.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x/metrics.go b/x/metrics.go index 819d1762bb1..039c8f717e3 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -461,8 +461,9 @@ func MonitorMemoryMetrics(lc *z.Closer) { defer fastTicker.Stop() update := func() { - // ReadMemStats stops the world, so don't call it too frequently. - // Calling it every minute is OK. + // ReadMemStats stops the world which is expensive especially when the + // heap is large. So don't call it too frequently. Calling it every + // minute is OK. var ms runtime.MemStats runtime.ReadMemStats(&ms) From 7eb1aeb459ef5b1ff7c7972b911407f21ef6a543 Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Wed, 2 Dec 2020 22:24:57 -0800 Subject: [PATCH 3/4] Update comment. --- x/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/metrics.go b/x/metrics.go index 039c8f717e3..46a32845916 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -70,7 +70,7 @@ var ( // PendingProposals records the current number of pending RAFT proposals. PendingProposals = stats.Int64("pending_proposals_total", "Number of pending proposals", stats.UnitDimensionless) - // MemoryInUse records the amount of memory allocated via jemalloc/Go + // MemoryAlloc records the amount of memory allocated via jemalloc MemoryAlloc = stats.Int64("memory_alloc_bytes", "Amount of memory allocated", stats.UnitBytes) // MemoryInUse records the current amount of used memory by Dgraph. From 0078f64cd5fe4c8f8ff587c6c22d8f182c824ddd Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Thu, 3 Dec 2020 08:57:16 -0800 Subject: [PATCH 4/4] Add test. --- dgraph/cmd/alpha/metrics_test.go | 1 + x/metrics.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dgraph/cmd/alpha/metrics_test.go b/dgraph/cmd/alpha/metrics_test.go index 85a21ad5189..e1d1ae44abb 100644 --- a/dgraph/cmd/alpha/metrics_test.go +++ b/dgraph/cmd/alpha/metrics_test.go @@ -96,6 +96,7 @@ func TestMetrics(t *testing.T) { // Dgraph Memory Metrics "dgraph_memory_idle_bytes", "dgraph_memory_inuse_bytes", "dgraph_memory_proc_bytes", + "dgraph_memory_alloc_bytes", // Dgraph Activity Metrics "dgraph_active_mutations_total", "dgraph_pending_proposals_total", "dgraph_pending_queries_total", diff --git a/x/metrics.go b/x/metrics.go index 46a32845916..15b394ae649 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -482,16 +482,20 @@ func MonitorMemoryMetrics(lc *z.Closer) { MemoryIdle.M(int64(idle)), MemoryProc.M(int64(getMemUsage()))) } + updateAlloc := func() { + ostats.Record(context.Background(), MemoryAlloc.M(z.NumAllocBytes())) + } // Call update immediately so that Dgraph reports memory stats without // having to wait for the first tick. update() + updateAlloc() for { select { case <-lc.HasBeenClosed(): return case <-fastTicker.C: - ostats.Record(context.Background(), MemoryAlloc.M(z.NumAllocBytes())) + updateAlloc() case <-ticker.C: update() }