feat sicedar: added readiness prober

CHANGELOG.md

@@ -15,6 +15,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 - [#1358](https://github.com/thanos-io/thanos/pull/1358) Added `part_size` configuration option for HTTP multipart requests minimum part size for S3 storage type
 - [#1363](https://github.com/thanos-io/thanos/pull/1363) Thanos Receive now exposes `thanos_receive_hashring_nodes` and `thanos_receive_hashring_tenants` metrics to monitor status of hash-rings
+- [#1395](https://github.com/thanos-io/thanos/pull/1395) Added `/-/ready` and `/-/healthy` endpoints to Thanos sidecar.
 
 ### Changed
 

cmd/thanos/compact.go

@@ -70,8 +70,7 @@ func (cs compactionSet) maxLevel() int {
 }
 
 func registerCompact(m map[string]setupFunc, app *kingpin.Application) {
- comp := component.Compact
- cmd := app.Command(comp.String(), "continuously compacts blocks in an object store bucket")
+ cmd := app.Command(component.Compact.String(), "continuously compacts blocks in an object store bucket")
 
  haltOnError := cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected.").
  Hidden().Default("true").Bool()
@@ -112,7 +111,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) {
  compactionConcurrency := cmd.Flag("compact.concurrency", "Number of goroutines to use when compacting groups.").
  Default("1").Int()
 
- m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
+ m[component.Compact.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
  return runCompact(g, logger, reg,
  *httpAddr,
  *dataDir,
@@ -127,7 +126,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) {
  compact.ResolutionLevel5m: time.Duration(*retention5m),
  compact.ResolutionLevel1h: time.Duration(*retention1h),
  },
- comp,
+ component.Compact,
  *disableDownsampling,
  *maxCompactionLevel,
  *blockSyncConcurrency,
@@ -170,9 +169,9 @@ func runCompact(
 
  downsampleMetrics := newDownsampleMetrics(reg)
 
- readinessProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+ statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
  // Initiate default HTTP listener providing metrics endpoint and readiness/liveness probes.
- if err := defaultHTTPListener(g, logger, reg, httpBindAddr, readinessProber); err != nil {
+ if err := defaultHTTPListener(g, logger, reg, httpBindAddr, statusProber); err != nil {
  return errors.Wrap(err, "create readiness prober")
  }
 
@@ -327,7 +326,7 @@ func runCompact(
  })
 
  level.Info(logger).Log("msg", "starting compact node")
- readinessProber.SetReady()
+ statusProber.SetReady()
  return nil
 }
 

cmd/thanos/main.go

@@ -70,7 +70,7 @@ func main() {
  tracingConfig := regCommonTracingFlags(app)
 
  cmds := map[string]setupFunc{}
- registerSidecar(cmds, app, "sidecar")
+ registerSidecar(cmds, app)
  registerStore(cmds, app, "store")
  registerQuery(cmds, app, "query")
  registerRule(cmds, app, "rule")

cmd/thanos/sidecar.go

@@ -11,28 +11,29 @@ import (
  "github.com/go-kit/kit/log"
  "github.com/go-kit/kit/log/level"
  "github.com/oklog/run"
- opentracing "github.com/opentracing/opentracing-go"
+ "github.com/opentracing/opentracing-go"
  "github.com/pkg/errors"
  "github.com/prometheus/client_golang/prometheus"
  "github.com/prometheus/common/model"
  "github.com/prometheus/tsdb/labels"
  "github.com/thanos-io/thanos/pkg/block/metadata"
  "github.com/thanos-io/thanos/pkg/component"
  "github.com/thanos-io/thanos/pkg/objstore/client"
+ "github.com/thanos-io/thanos/pkg/prober"
  "github.com/thanos-io/thanos/pkg/promclient"
  "github.com/thanos-io/thanos/pkg/reloader"
  "github.com/thanos-io/thanos/pkg/runutil"
  "github.com/thanos-io/thanos/pkg/shipper"
  "github.com/thanos-io/thanos/pkg/store"
  "github.com/thanos-io/thanos/pkg/store/storepb"
  "google.golang.org/grpc"
- kingpin "gopkg.in/alecthomas/kingpin.v2"
+ "gopkg.in/alecthomas/kingpin.v2"
 )
 
 const waitForExternalLabelsTimeout = 10 * time.Minute
 
-func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name string) {
- cmd := app.Command(name, "sidecar for Prometheus server")
+func registerSidecar(m map[string]setupFunc, app *kingpin.Application) {
+ cmd := app.Command(component.Sidecar.String(), "sidecar for Prometheus server")
 
  grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd)
 
@@ -54,7 +55,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri
 
  uploadCompacted := cmd.Flag("shipper.upload-compacted", "[Experimental] If true sidecar will try to upload compacted blocks as well. Useful for migration purposes. Works only if compaction is disabled on Prometheus.").Default("false").Hidden().Bool()
 
- m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
+ m[component.Sidecar.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
  rl := reloader.New(
  log.With(logger, "component", "reloader"),
  reloader.ReloadURLFromBase(*promURL),
@@ -77,6 +78,7 @@ func registerSidecar(m map[string]setupFunc, app *kingpin.Application, name stri
  objStoreConfig,
  rl,
  *uploadCompacted,
+ component.Sidecar,
  )
  }
 }
@@ -96,6 +98,7 @@ func runSidecar(
  objStoreConfig *pathOrContent,
  reloader *reloader.Reloader,
  uploadCompacted bool,
+ comp component.Component,
 ) error {
  var m = &promMetadata{
  promURL: promURL,
@@ -117,6 +120,12 @@ func runSidecar(
  uploads = false
  }
 
+ statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+ // Initiate default HTTP listener providing metrics endpoint and readiness/liveness probes.
+ if err := defaultHTTPListener(g, logger, reg, httpBindAddr, statusProber); err != nil {
+ return errors.Wrap(err, "create readiness prober")
+ }
+
  // Setup all the concurrent groups.
  {
  promUp := prometheus.NewGauge(prometheus.GaugeOpts{
@@ -148,6 +157,7 @@ func runSidecar(
  "err", err,
  )
  promUp.Set(0)
+ statusProber.SetNotReady(err)
  return err
  }
 
@@ -156,6 +166,7 @@ func runSidecar(
  "external_labels", m.Labels().String(),
  )
  promUp.Set(1)
+ statusProber.SetReady()
  lastHeartbeat.Set(float64(time.Now().UnixNano()) / 1e9)
  return nil
  })
@@ -195,9 +206,7 @@ func runSidecar(
  cancel()
  })
  }
- if err := metricHTTPListenGroup(g, logger, reg, httpBindAddr); err != nil {
- return err
- }
+
  {
  l, err := net.Listen("tcp", grpcBindAddr)
  if err != nil {

pkg/prober/prober.go

@@ -37,6 +37,19 @@ type Prober struct {
 func NewProber(component component.Component, logger log.Logger, reg prometheus.Registerer) *Prober {
  initialErr := fmt.Errorf(initialErrorFmt, component)
 
+ // From Kubernetes documentation https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ :
+ //
+ // liveness: Many applications running for long periods of time eventually transition to broken states,
+ // (healthy) and cannot recover except by being restarted.
+ // Kubernetes provides liveness probes to detect and remedy such situations.
+ //
+ // readiness: Sometimes, applications are temporarily unable to serve traffic.
+ // (ready) For example, an application might need to load large data or configuration files during startup,
+ // or depend on external services after startup. In such cases, you don’t want to kill the application,
+ // but you don’t want to send it requests either. Kubernetes provides readiness probes to detect
+ // and mitigate these situations. A pod with containers reporting that they are not ready
+ // does not receive traffic through Kubernetes Services.
+
  p := &Prober{
  component: component,
  logger: logger,

tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar-lts.yaml

@@ -145,6 +145,14 @@ spec:
  containerPort: 10902
  - name: grpc
  containerPort: 10901
+ livenessProbe:
+ httpGet:
+ port: 10902
+ path: /-/healthy
+ readinessProbe:
+ httpGet:
+ port: 10902
+ path: /-/ready
  volumeMounts:
  - name: prometheus
  mountPath: /var/prometheus

tutorials/kubernetes-demo/manifests/prometheus-ha-sidecar.yaml

@@ -129,6 +129,14 @@ spec:
  containerPort: 10902
  - name: grpc
  containerPort: 10901
+ livenessProbe:
+ httpGet:
+ port: 10902
+ path: /-/healthy
+ readinessProbe:
+ httpGet:
+ port: 10902
+ path: /-/ready
  volumeMounts:
  - name: prometheus
  mountPath: /var/prometheus