Merge remote-tracking branch 'origin/master' into add_hash_method

Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>

CHANGELOG.md

@@ -47,7 +47,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 - [#2893](https://github.com/thanos-io/thanos/pull/2893) Store: Rename metric `thanos_bucket_store_cached_postings_compression_time_seconds` to `thanos_bucket_store_cached_postings_compression_time_seconds_total`.
 - [#2915](https://github.com/thanos-io/thanos/pull/2915) Receive,Ruler: Enable TSDB directory locking by default. Add a new flag (`--tsdb.no-lockfile`) to override behavior.
 - [#2902](https://github.com/thanos-io/thanos/pull/2902) ui: React: Separate dedupe and partial response checkboxes per panel.
-- [#2931](https://github.com/thanos-io/thanos/pull/2931) Query: Allow passing a `storeMatcher[]` to select matching stores when debugging the querier. See [documentation](https://thanos.io/components/query.md/#store-filtering)
+- [#2931](https://github.com/thanos-io/thanos/pull/2931) Query: Allow passing a `storeMatch[]` to select matching stores when debugging the querier. See [documentation](https://thanos.io/components/query.md/#store-filtering)
 - [#2991](https://github.com/thanos-io/thanos/pull/2991) store: `operation` label value `getrange` changed to `get_range` for `thanos_store_bucket_cache_operation_requests_total` and `thanos_store_bucket_cache_operation_hits_total` to be consistent with bucket operation metrics.
 - [#2876](https://github.com/thanos-io/thanos/pull/2876) Receive,Ruler: Updated TSDB and switched to ChunkIterators instead of sample one, which avoids unnecessary decoding / encoding.
 

cmd/thanos/query-frontend.go

@@ -7,6 +7,7 @@ import (
  "net/http"
  "time"
 
+ "github.com/NYTimes/gziphandler"
  "github.com/cortexproject/cortex/pkg/querier/frontend"
  "github.com/cortexproject/cortex/pkg/querier/queryrange"
  "github.com/go-kit/kit/log"
@@ -22,10 +23,14 @@ import (
  "github.com/thanos-io/thanos/pkg/component"
  "github.com/thanos-io/thanos/pkg/extflag"
  "github.com/thanos-io/thanos/pkg/extprom"
+ extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
+ "github.com/thanos-io/thanos/pkg/logging"
  "github.com/thanos-io/thanos/pkg/prober"
  "github.com/thanos-io/thanos/pkg/queryfrontend"
  "github.com/thanos-io/thanos/pkg/queryfrontend/cache"
  httpserver "github.com/thanos-io/thanos/pkg/server/http"
+ "github.com/thanos-io/thanos/pkg/server/http/middleware"
+ "github.com/thanos-io/thanos/pkg/tracing"
 )
 
 type queryFrontendConfig struct {
@@ -35,6 +40,8 @@ type queryFrontendConfig struct {
  downstreamURL string
  compressResponses bool
  LogQueriesLongerThan time.Duration
+
+ requestLoggingDecision string
 }
 
 type queryRangeConfig struct {
@@ -77,6 +84,8 @@ func (c *queryFrontendConfig) registerFlag(cmd *kingpin.CmdClause) {
 
  cmd.Flag("query-frontend.log_queries_longer_than", "Log queries that are slower than the specified duration. "+
  "Set to 0 to disable. Set to < 0 to enable on all queries.").Default("0").DurationVar(&c.LogQueriesLongerThan)
+
+ cmd.Flag("log.request.decision", "Request Logging for logging the start and end of requests. LogFinishCall is enabled by default. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("LogFinishCall").EnumVar(&c.requestLoggingDecision, "NoLogCall", "LogFinishCall", "LogStartAndFinishCall")
 }
 
 func registerQueryFrontend(m map[string]setupFunc, app *kingpin.Application) {
@@ -85,15 +94,16 @@ func registerQueryFrontend(m map[string]setupFunc, app *kingpin.Application) {
  conf := &queryFrontendConfig{}
  conf.registerFlag(cmd)
 
- m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, _ opentracing.Tracer, _ <-chan struct{}, _ bool) error {
- return runQueryFrontend(g, logger, reg, conf, comp)
+ m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
+ return runQueryFrontend(g, logger, reg, tracer, conf, comp)
  }
 }
 
 func runQueryFrontend(
  g *run.Group,
  logger log.Logger,
  reg *prometheus.Registry,
+ tracer opentracing.Tracer,
  conf *queryFrontendConfig,
  comp component.Component,
 ) error {
@@ -153,21 +163,40 @@ func runQueryFrontend(
  prober.NewInstrumentation(comp, logger, extprom.WrapRegistererWithPrefix("thanos_", reg)),
  )
 
+ // Configure Request Logging for HTTP calls.
+ opts := []logging.Option{logging.WithDecider(func() logging.Decision {
+ return logging.LogDecision[conf.requestLoggingDecision]
+ })}
+ logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...)
+ ins := extpromhttp.NewInstrumentationMiddleware(reg)
+
  // Start metrics HTTP server.
  {
  srv := httpserver.New(logger, reg, comp, httpProbe,
  httpserver.WithListen(conf.http.bindAddress),
  httpserver.WithGracePeriod(time.Duration(conf.http.gracePeriod)),
  )
 
- injectf := func(f http.HandlerFunc) http.HandlerFunc {
+ instr := func(f http.HandlerFunc) http.HandlerFunc {
  hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- // Cortex frontend middlewares require orgID.
- f.ServeHTTP(w, r.WithContext(user.InjectOrgID(r.Context(), "fake")))
+ name := "query-frontend"
+ ins.NewHandler(
+ name,
+ logMiddleware.HTTPMiddleware(
+ name,
+ tracing.HTTPMiddleware(
+ tracer,
+ name,
+ logger,
+ gziphandler.GzipHandler(middleware.RequestID(f)),
+ ),
+ ),
+ // Cortex frontend middlewares require orgID.
+ ).ServeHTTP(w, r.WithContext(user.InjectOrgID(r.Context(), "fake")))
  })
  return hf
  }
- srv.Handle("/", injectf(fe.Handler().ServeHTTP))
+ srv.Handle("/", instr(fe.Handler().ServeHTTP))
 
  g.Add(func() error {
  statusProber.Healthy()

cmd/thanos/receive.go

@@ -379,6 +379,7 @@ func runReceive(
  if cw != nil {
  // Check the hashring configuration on before running the watcher.
  if err := cw.ValidateConfig(); err != nil {
+ cw.Stop()
  close(updates)
  return errors.Wrap(err, "failed to validate hashring configuration file")
  }

docs/components/query-frontend.md

@@ -125,5 +125,12 @@ Flags:
  Log queries that are slower than the specified
  duration. Set to 0 to disable. Set to < 0 to
  enable on all queries.
+ --log.request.decision=LogFinishCall
+ Request Logging for logging the start and end of
+ requests. LogFinishCall is enabled by default.
+ LogFinishCall : Logs the finish call of the
+ requests. LogStartAndFinishCall : Logs the start
+ and finish call of the requests. NoLogCall :
+ Disable request logging.
 
 ```

docs/components/query.md

@@ -226,7 +226,7 @@ Keep in mind that the maximum number of concurrent queries that are handled by q
 
 It's possible to provide a set of matchers to the Querier api to select specific stores to be used during the query using the `storeMatch[]` parameter. It is useful when debugging a slow/broken store.
 It uses the same format as the matcher of [Prometheus' federate api](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers).
-Note that at the moment the querier only supports the `__address__` which contain the address of the store as it is shown on the `/stores` endoint of the UI.
+Note that at the moment the querier only supports the `__address__` which contain the address of the store as it is shown on the `/stores` endpoint of the UI.
 
 Example:
 ```
@@ -236,7 +236,7 @@ Example:
 ```
 
 ```
-http://localhost:10901/api/v1/query?query=up&dedup=true&partial_response=true&storeMatch={__address__=~"prometheus-foo.*"}
+http://localhost:10901/api/v1/query?query=up&dedup=true&partial_response=true&storeMatch[]={__address__=~"prometheus-foo.*"}
 ```
 
 Will only return metrics from `prometheus-foo.thanos-sidecar:10901`