Store-gateway: add support to lazy mmap index-headers (#3431)

* Experimental lazy index-header reader Signed-off-by: Marco Pracucci <marco@pracucci.com> * Implemented lazy index-header reader Signed-off-by: Marco Pracucci <marco@pracucci.com> * Renamed CLI flags Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added copyright to new files Signed-off-by: Marco Pracucci <marco@pracucci.com> * Track metrics in the lazy index-header reader Signed-off-by: Marco Pracucci <marco@pracucci.com> * Ensure BucketStore.Close() is called in tests Signed-off-by: Marco Pracucci <marco@pracucci.com> * Ensure BucketStore.Close() is called in e2e tests too Signed-off-by: Marco Pracucci <marco@pracucci.com> * Shorten metric names Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added CHANGELOG entry Signed-off-by: Marco Pracucci <marco@pracucci.com> * Addressed review comments Signed-off-by: Marco Pracucci <marco@pracucci.com> * Removed readerTracker Signed-off-by: Marco Pracucci <marco@pracucci.com> * Fixed test and comments Signed-off-by: Marco Pracucci <marco@pracucci.com>
thanos-io · Nov 12, 2020 · 05fbe15 · 05fbe15
1 parent 6889ae8
commit 05fbe15
Show file tree

Hide file tree

Showing 13 changed files with 825 additions and 27 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#3277](https://github.com/thanos-io/thanos/pull/3277) Thanos Query: Introduce dynamic lookback interval. This allows queries with large step to make use of downsampled data.
 - [#3409](https://github.com/thanos-io/thanos/pull/3409) Compactor: Added support for no-compact-mark.json which excludes the block from compaction.
 - [#3245](https://github.com/thanos-io/thanos/pull/3245) Query Frontend: Add `query-frontend.org-id-header` flag to specify HTTP header(s) to populate slow query log (e.g. X-Grafana-User).
+- [#3431](https://github.com/thanos-io/thanos/pull/3431) Store: Added experimental support to lazy load index-headers at query time. When enabled via `--store.enable-index-header-lazy-reader` flag, the store-gateway will load into memory an index-header only once it's required at query time. Index-header will be automatically released after `--store.index-header-lazy-reader-idle-timeout` of inactivity.
+   * This, generally, reduces baseline memory usage of store when inactive, as well as a total number of mapped files (which is limited to 64k in some systems.
 
 ### Fixed
 

diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go
@@ -107,6 +107,12 @@ func registerStore(app *extkingpin.App) {
 		"Default is 24h, half of the default value for --delete-delay on compactor.").
 		Default("24h"))
 
+	lazyIndexReaderEnabled := cmd.Flag("store.enable-index-header-lazy-reader", "If true, Store Gateway will lazy memory map index-header only once the block is required by a query.").
+		Default("false").Bool()
+
+	lazyIndexReaderIdleTimeout := cmd.Flag("store.index-header-lazy-reader-idle-timeout", "If index-header lazy reader is enabled and this idle timeout setting is > 0, memory map-ed index-headers will be automatically released after 'idle timeout' inactivity.").
+		Hidden().Default("5m").Duration()
+
 	webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the bucket web UI interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos bucket web UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String()
 	webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String()
 
@@ -152,6 +158,8 @@ func registerStore(app *extkingpin.App) {
 			*postingOffsetsInMemSampling,
 			cachingBucketConfig,
 			getFlagsMap(cmd.Flags()),
+			*lazyIndexReaderEnabled,
+			*lazyIndexReaderIdleTimeout,
 		)
 	})
 }
@@ -184,6 +192,8 @@ func runStore(
 	postingOffsetsInMemSampling int,
 	cachingBucketConfig *extflag.PathOrContent,
 	flagsMap map[string]string,
+	lazyIndexReaderEnabled bool,
+	lazyIndexReaderIdleTimeout time.Duration,
 ) error {
 	grpcProbe := prober.NewGRPC()
 	httpProbe := prober.NewHTTP()
@@ -304,6 +314,8 @@ func runStore(
 		enablePostingsCompression,
 		postingOffsetsInMemSampling,
 		false,
+		lazyIndexReaderEnabled,
+		lazyIndexReaderIdleTimeout,
 	)
 	if err != nil {
 		return errors.Wrap(err, "create object storage store")

diff --git a/docs/components/store.md b/docs/components/store.md
@@ -171,6 +171,10 @@ Flags:
                                  before being deleted from bucket. Default is
                                  24h, half of the default value for
                                  --delete-delay on compactor.
+      --store.enable-index-header-lazy-reader
+                                 If true, Store Gateway will lazy memory map
+                                 index-header only once the block is required by
+                                 a query.
       --web.external-prefix=""   Static prefix for all HTML links and redirect
                                  URLs in the bucket web UI interface. Actual
                                  endpoints are still served on / or the

diff --git a/pkg/block/indexheader/binary_reader.go b/pkg/block/indexheader/binary_reader.go
@@ -637,8 +637,8 @@ func newBinaryTOCFromByteSlice(bs index.ByteSlice) (*BinaryTOC, error) {
 	}, nil
 }
 
-func (r BinaryReader) IndexVersion() int {
-	return r.indexVersion
+func (r BinaryReader) IndexVersion() (int, error) {
+	return r.indexVersion, nil
 }
 
 // TODO(bwplotka): Get advantage of multi value offset fetch.
@@ -871,7 +871,7 @@ func yoloString(b []byte) string {
 	return *((*string)(unsafe.Pointer(&b)))
 }
 
-func (r BinaryReader) LabelNames() []string {
+func (r BinaryReader) LabelNames() ([]string, error) {
 	allPostingsKeyName, _ := index.AllPostingsKey()
 	labelNames := make([]string, 0, len(r.postings))
 	for name := range r.postings {
@@ -882,7 +882,7 @@ func (r BinaryReader) LabelNames() []string {
 		labelNames = append(labelNames, name)
 	}
 	sort.Strings(labelNames)
-	return labelNames
+	return labelNames, nil
 }
 
 func (r *BinaryReader) Close() error { return r.c.Close() }

diff --git a/pkg/block/indexheader/header.go b/pkg/block/indexheader/header.go
@@ -18,7 +18,7 @@ type Reader interface {
 	io.Closer
 
 	// IndexVersion returns version of index.
-	IndexVersion() int
+	IndexVersion() (int, error)
 
 	// PostingsOffset returns start and end offsets of postings for given name and value.
 	// The end offset might be bigger than the actual posting ending, but not larger than the whole index file.
@@ -36,5 +36,5 @@ type Reader interface {
 	LabelValues(name string) ([]string, error)
 
 	// LabelNames returns all label names.
-	LabelNames() []string
+	LabelNames() ([]string, error)
 }
diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go
@@ -96,7 +96,7 @@ func TestReaders(t *testing.T) {
 
 			b := realByteSlice(indexFile.Bytes())
 
-			t.Run("binary", func(t *testing.T) {
+			t.Run("binary reader", func(t *testing.T) {
 				fn := filepath.Join(tmpDir, id.String(), block.IndexHeaderFilename)
 				testutil.Ok(t, WriteBinary(ctx, bkt, id, fn))
 
@@ -168,6 +168,18 @@ func TestReaders(t *testing.T) {
 
 				compareIndexToHeader(t, b, br)
 			})
+
+			t.Run("lazy binary reader", func(t *testing.T) {
+				fn := filepath.Join(tmpDir, id.String(), block.IndexHeaderFilename)
+				testutil.Ok(t, WriteBinary(ctx, bkt, id, fn))
+
+				br, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), nil, tmpDir, id, 3, NewLazyBinaryReaderMetrics(nil), nil)
+				testutil.Ok(t, err)
+
+				defer func() { testutil.Ok(t, br.Close()) }()
+
+				compareIndexToHeader(t, b, br)
+			})
 		})
 	}
 
@@ -178,7 +190,9 @@ func compareIndexToHeader(t *testing.T, indexByteSlice index.ByteSlice, headerRe
 	testutil.Ok(t, err)
 	defer func() { _ = indexReader.Close() }()
 
-	testutil.Equals(t, indexReader.Version(), headerReader.IndexVersion())
+	actVersion, err := headerReader.IndexVersion()
+	testutil.Ok(t, err)
+	testutil.Equals(t, indexReader.Version(), actVersion)
 
 	if indexReader.Version() == index.FormatV2 {
 		// For v2 symbols ref sequential integers 0, 1, 2 etc.
@@ -211,7 +225,9 @@ func compareIndexToHeader(t *testing.T, indexByteSlice index.ByteSlice, headerRe
 
 	expLabelNames, err := indexReader.LabelNames()
 	testutil.Ok(t, err)
-	testutil.Equals(t, expLabelNames, headerReader.LabelNames())
+	actualLabelNames, err := headerReader.LabelNames()
+	testutil.Ok(t, err)
+	testutil.Equals(t, expLabelNames, actualLabelNames)
 
 	expRanges, err := indexReader.PostingsRanges()
 	testutil.Ok(t, err)