prober: Remove spam of changing probe status (#5051)

* Remove spam of changing probe status

Signed-off-by: Juozas Vainauskas <vainauskasjuozas@gmail.com>

* Document changes in CHANGELOG.md

Signed-off-by: Juozas Vainauskas <vainauskasjuozas@gmail.com>

* Add status locks and adjust naming

Signed-off-by: Juozas Vainauskas <vainauskasjuozas@gmail.com>

* Fix linter warnings

Signed-off-by: Juozas Vainauskas <vainauskasjuozas@gmail.com>

CHANGELOG.md

@@ -24,6 +24,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Fixed
 
+- [#5051](https://github.com/thanos-io/thanos/pull/5051) Prober: Remove spam of changing probe status.
 - [#4918](https://github.com/thanos-io/thanos/pull/4918) Tracing: Fixing force tracing with Jaeger.
 - [#4879](https://github.com/thanos-io/thanos/pull/4879) Bucket verify: Fixed bug causing wrong number of blocks to be checked.
 - [#4908](https://github.com/thanos-io/thanos/pull/4908) UI: Show 'minus' icon and add tooltip when store min / max time is not available.

pkg/prober/intrumentation.go

@@ -4,6 +4,8 @@
 package prober
 
 import (
+ "sync"
+
  "github.com/go-kit/log"
  "github.com/go-kit/log/level"
  "github.com/prometheus/client_golang/prometheus"
@@ -13,8 +15,9 @@ import (
 )
 
 const (
- ready = "ready"
- healthy = "healthy"
+ ready = "ready"
+ notReady = "not-ready"
+ healthy = "healthy"
 )
 
 // InstrumentationProbe stores instrumentation state of Probe.
@@ -23,15 +26,17 @@ type InstrumentationProbe struct {
  component component.Component
  logger log.Logger
 
- status *prometheus.GaugeVec
+ statusMetric *prometheus.GaugeVec
+ mu sync.Mutex
+ statusString string
 }
 
 // NewInstrumentation returns InstrumentationProbe records readiness and healthiness for given component.
 func NewInstrumentation(component component.Component, logger log.Logger, reg prometheus.Registerer) *InstrumentationProbe {
  p := InstrumentationProbe{
  component: component,
  logger: logger,
- status: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+ statusMetric: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
  Name: "status",
  Help: "Represents status (0 indicates failure, 1 indicates success) of the component.",
  ConstLabels: map[string]string{"component": component.String()},
@@ -44,24 +49,34 @@ func NewInstrumentation(component component.Component, logger log.Logger, reg pr
 
 // Ready records the component status when Ready is called, if combined with other Probes.
 func (p *InstrumentationProbe) Ready() {
- p.status.WithLabelValues(ready).Set(1)
- level.Info(p.logger).Log("msg", "changing probe status", "status", "ready")
+ p.statusMetric.WithLabelValues(ready).Set(1)
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.statusString != ready {
+ level.Info(p.logger).Log("msg", "changing probe status", "status", ready)
+ p.statusString = ready
+ }
 }
 
 // NotReady records the component status when NotReady is called, if combined with other Probes.
 func (p *InstrumentationProbe) NotReady(err error) {
- p.status.WithLabelValues(ready).Set(0)
- level.Warn(p.logger).Log("msg", "changing probe status", "status", "not-ready", "reason", err)
+ p.statusMetric.WithLabelValues(ready).Set(0)
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.statusString != notReady {
+ level.Warn(p.logger).Log("msg", "changing probe status", "status", notReady, "reason", err)
+ p.statusString = notReady
+ }
 }
 
 // Healthy records the component status when Healthy is called, if combined with other Probes.
 func (p *InstrumentationProbe) Healthy() {
- p.status.WithLabelValues(healthy).Set(1)
+ p.statusMetric.WithLabelValues(healthy).Set(1)
  level.Info(p.logger).Log("msg", "changing probe status", "status", "healthy")
 }
 
 // NotHealthy records the component status when NotHealthy is called, if combined with other Probes.
 func (p *InstrumentationProbe) NotHealthy(err error) {
- p.status.WithLabelValues(healthy).Set(0)
+ p.statusMetric.WithLabelValues(healthy).Set(0)
  level.Info(p.logger).Log("msg", "changing probe status", "status", "not-healthy", "reason", err)
 }