*: add metrics to the reloader package

CHANGELOG.md

@@ -18,6 +18,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ### Added
 
 - [#2502](https://github.com/thanos-io/thanos/pull/2502) Added `hints` field to `SeriesResponse`. Hints in an opaque data structure that can be used to carry additional information from the store and its content is implementation specific.
+- [#2521](https://github.com/thanos-io/thanos/pull/2521) Sidecar: add `thanos_sidecar_reloader_reloads_failed_total`, `thanos_sidecar_reloader_reloads_total`, `thanos_sidecar_reloader_watch_errors_total`, `thanos_sidecar_reloader_watch_events_total` and `thanos_sidecar_reloader_watches` metrics.
 
 ### Changed
 

cmd/thanos/sidecar.go

@@ -80,6 +80,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application) {
 	m[component.Sidecar.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
 		rl := reloader.New(
 			log.With(logger, "component", "reloader"),
+			extprom.WrapRegistererWithPrefix("thanos_sidecar_", reg),
 			reloader.ReloadURLFromBase(*promURL),
 			*reloaderCfgFile,
 			*reloaderCfgOutputFile,

pkg/reloader/example_test.go

@@ -20,6 +20,7 @@ func ExampleReloader() {
 		log.Fatal(err)
 	}
 	rl := reloader.New(
+		nil,
 		nil,
 		reloader.ReloadURLFromBase(u),
 		"/path/to/cfg",

pkg/reloader/reloader.go

@@ -72,6 +72,8 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/thanos-io/thanos/pkg/runutil"
 )
 
@@ -89,6 +91,12 @@ type Reloader struct {
 
 	lastCfgHash  []byte
 	lastRuleHash []byte
+
+	reloads      prometheus.Counter
+	reloadErrors prometheus.Counter
+	watches      prometheus.Gauge
+	watchEvents  prometheus.Counter
+	watchErrors  prometheus.Counter
 }
 
 var firstGzipBytes = []byte{0x1f, 0x8b, 0x08}
@@ -98,19 +106,51 @@ var firstGzipBytes = []byte{0x1f, 0x8b, 0x08}
 // If cfgOutputFile is not empty the config file will be decompressed if needed, environment variables
 // will be substituted and the output written into the given path. Prometheus should then use
 // cfgOutputFile as its config file path.
-func New(logger log.Logger, reloadURL *url.URL, cfgFile string, cfgOutputFile string, ruleDirs []string) *Reloader {
+func New(logger log.Logger, reg prometheus.Registerer, reloadURL *url.URL, cfgFile string, cfgOutputFile string, ruleDirs []string) *Reloader {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
-	return &Reloader{
+	r := &Reloader{
 		logger:        logger,
 		reloadURL:     reloadURL,
 		cfgFile:       cfgFile,
 		cfgOutputFile: cfgOutputFile,
 		ruleDirs:      ruleDirs,
 		watchInterval: 3 * time.Minute,
 		retryInterval: 5 * time.Second,
+
+		reloads: promauto.With(reg).NewCounter(
+			prometheus.CounterOpts{
+				Name: "reloader_reloads_total",
+				Help: "Total number of reload requests.",
+			},
+		),
+		reloadErrors: promauto.With(reg).NewCounter(
+			prometheus.CounterOpts{
+				Name: "reloader_reloads_failed_total",
+				Help: "Total number of reload requests that failed.",
+			},
+		),
+		watches: promauto.With(reg).NewGauge(
+			prometheus.GaugeOpts{
+				Name: "reloader_watches",
+				Help: "Number of resources watched by the reloader.",
+			},
+		),
+		watchEvents: promauto.With(reg).NewCounter(
+			prometheus.CounterOpts{
+				Name: "reloader_watch_events_total",
+				Help: "Total number of events received by the reloader from the watcher.",
+			},
+		),
+		watchErrors: promauto.With(reg).NewCounter(
+			prometheus.CounterOpts{
+				Name: "reloader_watch_errors_total",
+				Help: "Total number of errors received by the reloader from the watcher.",
+			},
+		),
 	}
+	return r
 }
 
 // We cannot detect everything via watch. Watch interval controls how often we re-read given dirs non-recursively.
@@ -154,6 +194,7 @@ func (r *Reloader) Watch(ctx context.Context) error {
 	tick := time.NewTicker(r.watchInterval)
 	defer tick.Stop()
 
+	r.watches.Set(float64(len(watchables)))
 	level.Info(r.logger).Log(
 		"msg", "started watching config file and non-recursively rule dirs for changes",
 		"cfg", r.cfgFile,
@@ -166,11 +207,12 @@ func (r *Reloader) Watch(ctx context.Context) error {
 			return nil
 		case <-tick.C:
 		case event := <-watcher.Events:
-			// TODO(bwplotka): Add metric if we are not cycling CPU here too much.
+			r.watchEvents.Inc()
 			if _, ok := watchables[filepath.Dir(event.Name)]; !ok {
 				continue
 			}
 		case err := <-watcher.Errors:
+			r.watchErrors.Inc()
 			level.Error(r.logger).Log("msg", "watch error", "err", err)
 			continue
 		}
@@ -280,7 +322,9 @@ func (r *Reloader) apply(ctx context.Context) error {
 	defer cancel()
 
 	if err := runutil.RetryWithLog(r.logger, r.retryInterval, retryCtx.Done(), func() error {
+		r.reloads.Inc()
 		if err := r.triggerReload(ctx); err != nil {
+			r.reloadErrors.Inc()
 			return errors.Wrap(err, "trigger reload")
 		}
 

pkg/reloader/reloader_test.go

@@ -62,7 +62,7 @@ func TestReloader_ConfigApply(t *testing.T) {
 		input  = path.Join(dir, "in", "cfg.yaml.tmpl")
 		output = path.Join(dir, "out", "cfg.yaml")
 	)
-	reloader := New(nil, reloadURL, input, output, nil)
+	reloader := New(nil, nil, reloadURL, input, output, nil)
 	reloader.watchInterval = 9999 * time.Hour // Disable interval to test watch logic only.
 	reloader.retryInterval = 100 * time.Millisecond
 
@@ -205,7 +205,7 @@ func TestReloader_RuleApply(t *testing.T) {
 	testutil.Ok(t, os.Mkdir(path.Join(dir2, "rule-dir"), os.ModePerm))
 	testutil.Ok(t, os.Symlink(path.Join(dir2, "rule-dir"), path.Join(dir, "rule-dir")))
 
-	reloader := New(nil, reloadURL, "", "", []string{dir, path.Join(dir, "rule-dir")})
+	reloader := New(nil, nil, reloadURL, "", "", []string{dir, path.Join(dir, "rule-dir")})
 	reloader.watchInterval = 100 * time.Millisecond
 	reloader.retryInterval = 100 * time.Millisecond