Store: Add tenant label to exported metrics

CHANGELOG.md

@@ -18,6 +18,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message.
 - [#6760](https://github.com/thanos-io/thanos/pull/6760) Query Frontend: Added TLS support in `--query-frontend.downstream-tripper-config` and `--query-frontend.downstream-tripper-config-file`
 - [#6749](https://github.com/thanos-io/thanos/pull/6308) Store Gateway: Added `thanos_store_index_cache_fetch_duration_seconds` histogram for tracking latency of fetching data from index cache.
+- [#6690](https://github.com/thanos-io/thanos/pull/6690) Store: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing dashboard queries to be incorrect due to the added label.
+- [#6530](https://github.com/thanos-io/thanos/pull/6530) / [#6690](https://github.com/thanos-io/thanos/pull/6690) Query: Add command line arguments for configuring tenants and forward tenant information to Store Gateway.
 
 ### Changed
 

cmd/thanos/query.go

@@ -217,9 +217,9 @@ func registerQuery(app *extkingpin.App) {
  queryTelemetrySamplesQuantiles := cmd.Flag("query.telemetry.request-samples-quantiles", "The quantiles for exporting metrics about the samples count quantiles.").Default("100", "1000", "10000", "100000", "1000000").Float64List()
  queryTelemetrySeriesQuantiles := cmd.Flag("query.telemetry.request-series-seconds-quantiles", "The quantiles for exporting metrics about the series count quantiles.").Default("10", "100", "1000", "10000", "100000").Float64List()
 
- tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).Hidden().String()
- defaultTenant := cmd.Flag("query.default-tenant", "Name of the default tenant.").Default(tenancy.DefaultTenant).Hidden().String()
- tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Hidden().Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName)
+ tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).String()
+ defaultTenant := cmd.Flag("query.default-tenant-id", "Default tenant ID to use if tenant header is not present").Default(tenancy.DefaultTenant).String()
+ tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName)
 
  var storeRateLimits store.SeriesSelectLimits
  storeRateLimits.RegisterFlags(cmd)

docs/components/query.md

@@ -360,6 +360,9 @@ Flags:
  = max(rangeSeconds / 250, defaultStep)).
  This will not work from Grafana, but Grafana
  has __step variable which can be used.
+ --query.default-tenant-id="default-tenant"
+ Default tenant ID to use if tenant header is
+ not present
  --query.lookback-delta=QUERY.LOOKBACK-DELTA
  The maximum lookback duration for retrieving
  metrics during expression evaluations.
@@ -404,6 +407,14 @@ Flags:
  --query.telemetry.request-series-seconds-quantiles=10... ...
  The quantiles for exporting metrics about the
  series count quantiles.
+ --query.tenant-certificate-field=
+ Use TLS client's certificate field to determine
+ tenant for write requests. Must be one of
+ organization, organizationalUnit or commonName.
+ This setting will cause the query.tenant-header
+ flag value to be ignored.
+ --query.tenant-header="THANOS-TENANT"
+ HTTP header to determine tenant.
  --query.timeout=2m Maximum time to process query by query node.
  --request.logging-config=<content>
  Alternative to 'request.logging-config-file'

pkg/promclient/promclient.go

@@ -105,7 +105,7 @@ func NewWithTracingClient(logger log.Logger, httpClient *http.Client, userAgent
 
 // req2xx sends a request to the given url.URL. If method is http.MethodPost then
 // the raw query is encoded in the body and the appropriate Content-Type is set.
-func (c *Client) req2xx(ctx context.Context, u *url.URL, method string) (_ []byte, _ int, err error) {
+func (c *Client) req2xx(ctx context.Context, u *url.URL, method string, headers http.Header) (_ []byte, _ int, err error) {
  var b io.Reader
  if method == http.MethodPost {
  rq := u.RawQuery
@@ -117,6 +117,10 @@ func (c *Client) req2xx(ctx context.Context, u *url.URL, method string) (_ []byt
  if err != nil {
  return nil, 0, errors.Wrapf(err, "create %s request", method)
  }
+ if headers != nil {
+ req.Header = headers
+ }
+
  if c.userAgent != "" {
  req.Header.Set("User-Agent", c.userAgent)
  }
@@ -166,7 +170,7 @@ func (c *Client) ExternalLabels(ctx context.Context, base *url.URL) (labels.Labe
  span, ctx := tracing.StartSpan(ctx, "/prom_config HTTP[client]")
  defer span.Finish()
 
- body, _, err := c.req2xx(ctx, &u, http.MethodGet)
+ body, _, err := c.req2xx(ctx, &u, http.MethodGet, nil)
  if err != nil {
  return nil, err
  }
@@ -363,6 +367,7 @@ type QueryOptions struct {
  MaxSourceResolution string
  Engine string
  Explain bool
+ HTTPHeaders http.Header
 }
 
 func (p *QueryOptions) AddTo(values url.Values) error {
@@ -423,7 +428,7 @@ func (c *Client) QueryInstant(ctx context.Context, base *url.URL, query string,
  method = http.MethodGet
  }
 
- body, _, err := c.req2xx(ctx, &u, method)
+ body, _, err := c.req2xx(ctx, &u, method, opts.HTTPHeaders)
  if err != nil {
  return nil, nil, nil, errors.Wrap(err, "read query instant response")
  }
@@ -529,7 +534,7 @@ func (c *Client) QueryRange(ctx context.Context, base *url.URL, query string, st
  span, ctx := tracing.StartSpan(ctx, "/prom_query_range HTTP[client]")
  defer span.Finish()
 
- body, _, err := c.req2xx(ctx, &u, http.MethodGet)
+ body, _, err := c.req2xx(ctx, &u, http.MethodGet, opts.HTTPHeaders)
  if err != nil {
  return nil, nil, nil, errors.Wrap(err, "read query range response")
  }
@@ -612,7 +617,7 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode
  span, ctx := tracing.StartSpan(ctx, "/alertmanager_alerts HTTP[client]")
  defer span.Finish()
 
- body, _, err := c.req2xx(ctx, &u, http.MethodGet)
+ body, _, err := c.req2xx(ctx, &u, http.MethodGet, nil)
  if err != nil {
  return nil, err
  }
@@ -643,7 +648,7 @@ func (c *Client) BuildVersion(ctx context.Context, base *url.URL) (string, error
  defer span.Finish()
 
  // We get status code 404 or 405 for prometheus versions lower than 2.14.0
- body, code, err := c.req2xx(ctx, &u, http.MethodGet)
+ body, code, err := c.req2xx(ctx, &u, http.MethodGet, nil)
  if err != nil {
  if code == http.StatusNotFound {
  return "0", nil
@@ -675,7 +680,7 @@ func (c *Client) get2xxResultWithGRPCErrors(ctx context.Context, spanName string
  span, ctx := tracing.StartSpan(ctx, spanName)
  defer span.Finish()
 
- body, code, err := c.req2xx(ctx, u, http.MethodGet)
+ body, code, err := c.req2xx(ctx, u, http.MethodGet, nil)
  if err != nil {
  if code, exists := statusToCode[code]; exists && code != 0 {
  return status.Error(code, err.Error())