Store-gateway: add support to lazy mmap index-headers

CHANGELOG.md

@@ -28,6 +28,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#3277](https://github.com/thanos-io/thanos/pull/3277) Thanos Query: Introduce dynamic lookback interval. This allows queries with large step to make use of downsampled data.
 - [#3409](https://github.com/thanos-io/thanos/pull/3409) Compactor: Added support for no-compact-mark.json which excludes the block from compaction.
 - [#3245](https://github.com/thanos-io/thanos/pull/3245) Query Frontend: Add `query-frontend.org-id-header` flag to specify HTTP header(s) to populate slow query log (e.g. X-Grafana-User).
+- [#3431](https://github.com/thanos-io/thanos/pull/3431) Store: Added experimental support to lazy load index-headers at query time. When enabled via `--store.enable-index-header-lazy-reader` flag, the store-gateway will load into memory an index-header only once it's required at query time. Index-header will be automatically released after `--store.index-header-lazy-reader-idle-timeout` of inactivity.
+ * This, generally, reduces baseline memory usage of store when inactive, as well as a total number of mapped files (which is limited to 64k in some systems.
 
 ### Fixed
 

cmd/thanos/store.go

@@ -107,6 +107,12 @@ func registerStore(app *extkingpin.App) {
  "Default is 24h, half of the default value for --delete-delay on compactor.").
  Default("24h"))
 
+ lazyIndexReaderEnabled := cmd.Flag("store.enable-index-header-lazy-reader", "If true, Store Gateway will lazy memory map index-header only once the block is required by a query.").
+ Default("false").Bool()
+
+ lazyIndexReaderIdleTimeout := cmd.Flag("store.index-header-lazy-reader-idle-timeout", "If index-header lazy reader is enabled and this idle timeout setting is > 0, memory map-ed index-headers will be automatically released after 'idle timeout' inactivity.").
+ Hidden().Default("5m").Duration()
+
  webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the bucket web UI interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos bucket web UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String()
  webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String()
 
@@ -152,6 +158,8 @@ func registerStore(app *extkingpin.App) {
  *postingOffsetsInMemSampling,
  cachingBucketConfig,
  getFlagsMap(cmd.Flags()),
+ *lazyIndexReaderEnabled,
+ *lazyIndexReaderIdleTimeout,
  )
  })
 }
@@ -184,6 +192,8 @@ func runStore(
  postingOffsetsInMemSampling int,
  cachingBucketConfig *extflag.PathOrContent,
  flagsMap map[string]string,
+ lazyIndexReaderEnabled bool,
+ lazyIndexReaderIdleTimeout time.Duration,
 ) error {
  grpcProbe := prober.NewGRPC()
  httpProbe := prober.NewHTTP()
@@ -304,6 +314,8 @@ func runStore(
  enablePostingsCompression,
  postingOffsetsInMemSampling,
  false,
+ lazyIndexReaderEnabled,
+ lazyIndexReaderIdleTimeout,
  )
  if err != nil {
  return errors.Wrap(err, "create object storage store")

docs/components/store.md

@@ -171,6 +171,10 @@ Flags:
  before being deleted from bucket. Default is
  24h, half of the default value for
  --delete-delay on compactor.
+ --store.enable-index-header-lazy-reader
+ If true, Store Gateway will lazy memory map
+ index-header only once the block is required by
+ a query.
  --web.external-prefix="" Static prefix for all HTML links and redirect
  URLs in the bucket web UI interface. Actual
  endpoints are still served on / or the

pkg/block/indexheader/binary_reader.go

@@ -637,8 +637,8 @@ func newBinaryTOCFromByteSlice(bs index.ByteSlice) (*BinaryTOC, error) {
  }, nil
 }
 
-func (r BinaryReader) IndexVersion() int {
- return r.indexVersion
+func (r BinaryReader) IndexVersion() (int, error) {
+ return r.indexVersion, nil
 }
 
 // TODO(bwplotka): Get advantage of multi value offset fetch.
@@ -871,7 +871,7 @@ func yoloString(b []byte) string {
  return *((*string)(unsafe.Pointer(&b)))
 }
 
-func (r BinaryReader) LabelNames() []string {
+func (r BinaryReader) LabelNames() ([]string, error) {
  allPostingsKeyName, _ := index.AllPostingsKey()
  labelNames := make([]string, 0, len(r.postings))
  for name := range r.postings {
@@ -882,7 +882,7 @@ func (r BinaryReader) LabelNames() []string {
  labelNames = append(labelNames, name)
  }
  sort.Strings(labelNames)
- return labelNames
+ return labelNames, nil
 }
 
 func (r *BinaryReader) Close() error { return r.c.Close() }

pkg/block/indexheader/header.go

@@ -18,7 +18,7 @@ type Reader interface {
  io.Closer
 
  // IndexVersion returns version of index.
- IndexVersion() int
+ IndexVersion() (int, error)
 
  // PostingsOffset returns start and end offsets of postings for given name and value.
  // The end offset might be bigger than the actual posting ending, but not larger than the whole index file.
@@ -36,5 +36,5 @@ type Reader interface {
  LabelValues(name string) ([]string, error)
 
  // LabelNames returns all label names.
- LabelNames() []string
+ LabelNames() ([]string, error)
 }

pkg/block/indexheader/header_test.go

@@ -96,7 +96,7 @@ func TestReaders(t *testing.T) {
 
  b := realByteSlice(indexFile.Bytes())
 
- t.Run("binary", func(t *testing.T) {
+ t.Run("binary reader", func(t *testing.T) {
  fn := filepath.Join(tmpDir, id.String(), block.IndexHeaderFilename)
  testutil.Ok(t, WriteBinary(ctx, bkt, id, fn))
 
@@ -168,6 +168,18 @@ func TestReaders(t *testing.T) {
 
  compareIndexToHeader(t, b, br)
  })
+
+ t.Run("lazy binary reader", func(t *testing.T) {
+ fn := filepath.Join(tmpDir, id.String(), block.IndexHeaderFilename)
+ testutil.Ok(t, WriteBinary(ctx, bkt, id, fn))
+
+ br, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), nil, tmpDir, id, 3, NewLazyBinaryReaderMetrics(nil), nil)
+ testutil.Ok(t, err)
+
+ defer func() { testutil.Ok(t, br.Close()) }()
+
+ compareIndexToHeader(t, b, br)
+ })
  })
  }
 
@@ -178,7 +190,9 @@ func compareIndexToHeader(t *testing.T, indexByteSlice index.ByteSlice, headerRe
  testutil.Ok(t, err)
  defer func() { _ = indexReader.Close() }()
 
- testutil.Equals(t, indexReader.Version(), headerReader.IndexVersion())
+ actVersion, err := headerReader.IndexVersion()
+ testutil.Ok(t, err)
+ testutil.Equals(t, indexReader.Version(), actVersion)
 
  if indexReader.Version() == index.FormatV2 {
  // For v2 symbols ref sequential integers 0, 1, 2 etc.
@@ -211,7 +225,9 @@ func compareIndexToHeader(t *testing.T, indexByteSlice index.ByteSlice, headerRe
 
  expLabelNames, err := indexReader.LabelNames()
  testutil.Ok(t, err)
- testutil.Equals(t, expLabelNames, headerReader.LabelNames())
+ actualLabelNames, err := headerReader.LabelNames()
+ testutil.Ok(t, err)
+ testutil.Equals(t, expLabelNames, actualLabelNames)
 
  expRanges, err := indexReader.PostingsRanges()
  testutil.Ok(t, err)