From aaa4b85f9a77ab583a2e042adf69bda6c3a008d7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 12 Sep 2022 08:53:28 +0200 Subject: [PATCH 1/4] Capture reorg distance and depths --- beacon-chain/blockchain/head.go | 9 +++++++-- beacon-chain/blockchain/metrics.go | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/beacon-chain/blockchain/head.go b/beacon-chain/blockchain/head.go index f7a7cb27cf23..adca77894a46 100644 --- a/beacon-chain/blockchain/head.go +++ b/beacon-chain/blockchain/head.go @@ -106,15 +106,20 @@ func (s *Service) saveHead(ctx context.Context, newHeadRoot [32]byte, headBlock log.WithError(err).Error("Could not find common ancestor root") commonRoot = params.BeaconConfig().ZeroHash } + dis := headSlot + newHeadSlot - 2*forkSlot + dep := math.Max(uint64(headSlot-forkSlot), uint64(newHeadSlot-forkSlot)) log.WithFields(logrus.Fields{ "newSlot": fmt.Sprintf("%d", newHeadSlot), "newRoot": fmt.Sprintf("%#x", newHeadRoot), "oldSlot": fmt.Sprintf("%d", headSlot), "oldRoot": fmt.Sprintf("%#x", oldHeadRoot), "commonAncestorRoot": fmt.Sprintf("%#x", commonRoot), - "distance": headSlot + newHeadSlot - 2*forkSlot, - "depth": math.Max(uint64(headSlot-forkSlot), uint64(newHeadSlot-forkSlot)), + "distance": dis, + "depth": dep, }).Info("Chain reorg occurred") + reorgDistance.Set(float64(dis)) + reorgDepth.Set(float64(dep)) + isOptimistic, err := s.IsOptimistic(ctx) if err != nil { return errors.Wrap(err, "could not check if node is optimistically synced") diff --git a/beacon-chain/blockchain/metrics.go b/beacon-chain/blockchain/metrics.go index 20589be0cd0a..bd75783197d8 100644 --- a/beacon-chain/blockchain/metrics.go +++ b/beacon-chain/blockchain/metrics.go @@ -199,6 +199,16 @@ var ( Buckets: []float64{1, 5, 20, 100, 500, 1000}, }, ) + reorgDistance = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reorg_distance", + Help: "The distance of the reorg", + }, + ) + reorgDepth = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "reorg_depth", + Help: "The depth of the reorg", + }, + ) ) // reportSlotMetrics reports slot related metrics. From 0133d22dc830e239302b8b5b5f35ed409be055bc Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 12 Sep 2022 16:01:58 +0200 Subject: [PATCH 2/4] Use historgrams instead of gauges --- beacon-chain/blockchain/metrics.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/beacon-chain/blockchain/metrics.go b/beacon-chain/blockchain/metrics.go index bd75783197d8..732c4253da66 100644 --- a/beacon-chain/blockchain/metrics.go +++ b/beacon-chain/blockchain/metrics.go @@ -199,15 +199,19 @@ var ( Buckets: []float64{1, 5, 20, 100, 500, 1000}, }, ) - reorgDistance = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "reorg_distance", - Help: "The distance of the reorg", - }, + reorgDistance = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "reorg_distance", + Help: "Captures distance of reorgs", + Buckets: []float64{1, 2, 4, 8, 16, 32, 64}, + }, ) - reorgDepth = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "reorg_depth", - Help: "The depth of the reorg", - }, + reorgDepth = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "reorg_depth", + Help: "Captures depth of reorgs", + Buckets: []float64{1, 2, 4, 8, 16, 32}, + }, ) ) From f0ff50ad322c11171616bd2d3d0ba0e8492e9151 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 12 Sep 2022 16:05:52 +0200 Subject: [PATCH 3/4] Fix build --- beacon-chain/blockchain/head.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon-chain/blockchain/head.go b/beacon-chain/blockchain/head.go index adca77894a46..aa291feebcb8 100644 --- a/beacon-chain/blockchain/head.go +++ b/beacon-chain/blockchain/head.go @@ -117,8 +117,8 @@ func (s *Service) saveHead(ctx context.Context, newHeadRoot [32]byte, headBlock "distance": dis, "depth": dep, }).Info("Chain reorg occurred") - reorgDistance.Set(float64(dis)) - reorgDepth.Set(float64(dep)) + reorgDistance.Observe(float64(dis)) + reorgDepth.Observe(float64(dep)) isOptimistic, err := s.IsOptimistic(ctx) if err != nil { From 9a9973ed895b0046ceb4796557478f2f74408c5c Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 12 Sep 2022 16:43:07 +0200 Subject: [PATCH 4/4] Edit help texts --- beacon-chain/blockchain/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon-chain/blockchain/metrics.go b/beacon-chain/blockchain/metrics.go index 732c4253da66..59a4a92dd775 100644 --- a/beacon-chain/blockchain/metrics.go +++ b/beacon-chain/blockchain/metrics.go @@ -202,14 +202,14 @@ var ( reorgDistance = promauto.NewHistogram( prometheus.HistogramOpts{ Name: "reorg_distance", - Help: "Captures distance of reorgs", + Help: "Captures distance of reorgs. Distance is defined as the number of blocks between the old head and the new head", Buckets: []float64{1, 2, 4, 8, 16, 32, 64}, }, ) reorgDepth = promauto.NewHistogram( prometheus.HistogramOpts{ Name: "reorg_depth", - Help: "Captures depth of reorgs", + Help: "Captures depth of reorgs. Depth is defined as the number of blocks between the head and the common ancestor", Buckets: []float64{1, 2, 4, 8, 16, 32}, }, )