From 2c575215675d1c634636884b3932a42aea3b0e5f Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 13 Jun 2023 00:58:07 -0700 Subject: [PATCH 1/8] compactor set index stats on block Signed-off-by: Ben Ye --- pkg/block/metadata/meta.go | 12 +++++++++++ pkg/compact/compact.go | 42 +++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pkg/block/metadata/meta.go b/pkg/block/metadata/meta.go index 787a03c241..a444d3bbae 100644 --- a/pkg/block/metadata/meta.go +++ b/pkg/block/metadata/meta.go @@ -90,6 +90,18 @@ type Thanos struct { // Rewrites is present when any rewrite (deletion, relabel etc) were applied to this block. Optional. Rewrites []Rewrite `json:"rewrites,omitempty"` + + IndexStats IndexStats `json:"index_stats,omitempty"` +} + +type IndexStats struct { + SeriesMinSize int64 `json:"series_min_size,omitempty"` + SeriesMaxSize int64 `json:"series_max_size,omitempty"` + SeriesAvgSize int64 `json:"series_avg_size,omitempty"` + + ChunkMinSize int64 `json:"chunk_min_size,omitempty"` + ChunkMaxSize int64 `json:"chunk_max_size,omitempty"` + ChunkAvgSize int64 `json:"chunk_avg_size,omitempty"` } type Rewrite struct { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index 585d5d6b4d..c6393e9891 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -818,7 +818,7 @@ func outOfOrderChunkError(err error, brokenBlock ulid.ULID) OutOfOrderChunksErro return OutOfOrderChunksError{err: err, id: brokenBlock} } -// IsOutOfOrderChunk returns true if the base error is a OutOfOrderChunkError. +// IsOutOfOrderChunkError returns true if the base error is a OutOfOrderChunkError. func IsOutOfOrderChunkError(err error) bool { _, ok := errors.Cause(err).(OutOfOrderChunksError) return ok @@ -1100,28 +1100,46 @@ func (cg *Group) compact(ctx context.Context, dir string, planner Planner, comp bdir := filepath.Join(dir, compID.String()) index := filepath.Join(bdir, block.IndexFilename) - newMeta, err := metadata.InjectThanos(cg.logger, bdir, metadata.Thanos{ - Labels: cg.labels.Map(), - Downsample: metadata.ThanosDownsample{Resolution: cg.resolution}, - Source: metadata.CompactorSource, - SegmentFiles: block.GetSegmentFiles(bdir), - }, nil) - if err != nil { - return false, ulid.ULID{}, errors.Wrapf(err, "failed to finalize the block %s", bdir) + if err := os.Remove(filepath.Join(bdir, "tombstones")); err != nil { + return false, ulid.ULID{}, errors.Wrap(err, "remove tombstones") } - if err = os.Remove(filepath.Join(bdir, "tombstones")); err != nil { - return false, ulid.ULID{}, errors.Wrap(err, "remove tombstones") + newMeta, err := metadata.ReadFromDir(bdir) + if err != nil { + return false, ulid.ULID{}, errors.Wrap(err, "read new meta") } + var stats block.HealthStats // Ensure the output block is valid. err = tracing.DoInSpanWithErr(ctx, "compaction_verify_index", func(ctx context.Context) error { - return block.VerifyIndex(cg.logger, index, newMeta.MinTime, newMeta.MaxTime) + stats, err = block.GatherIndexHealthStats(cg.logger, index, newMeta.MinTime, newMeta.MaxTime) + if err != nil { + return err + } + return stats.AnyErr() }) if !cg.acceptMalformedIndex && err != nil { return false, ulid.ULID{}, halt(errors.Wrapf(err, "invalid result block %s", bdir)) } + newMeta, err = metadata.InjectThanos(cg.logger, bdir, metadata.Thanos{ + Labels: cg.labels.Map(), + Downsample: metadata.ThanosDownsample{Resolution: cg.resolution}, + Source: metadata.CompactorSource, + SegmentFiles: block.GetSegmentFiles(bdir), + IndexStats: metadata.IndexStats{ + SeriesMinSize: stats.SeriesMinSize, + SeriesMaxSize: stats.SeriesMaxSize, + SeriesAvgSize: stats.SeriesAvgSize, + ChunkMinSize: stats.ChunkMinSize, + ChunkMaxSize: stats.ChunkMaxSize, + ChunkAvgSize: stats.ChunkAvgSize, + }, + }, nil) + if err != nil { + return false, ulid.ULID{}, errors.Wrapf(err, "failed to finalize the block %s", bdir) + } + // Ensure the output block is not overlapping with anything else, // unless vertical compaction is enabled. if !cg.enableVerticalCompaction { From ba165df763eade8641c920d14f5944095140ff7b Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 13 Jun 2023 01:04:33 -0700 Subject: [PATCH 2/8] comment Signed-off-by: Ben Ye --- pkg/block/metadata/meta.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/block/metadata/meta.go b/pkg/block/metadata/meta.go index a444d3bbae..4ddf924d39 100644 --- a/pkg/block/metadata/meta.go +++ b/pkg/block/metadata/meta.go @@ -91,6 +91,7 @@ type Thanos struct { // Rewrites is present when any rewrite (deletion, relabel etc) were applied to this block. Optional. Rewrites []Rewrite `json:"rewrites,omitempty"` + // IndexStats contains stats info related to block index. IndexStats IndexStats `json:"index_stats,omitempty"` } From 06be88369b765febce4ba0472fe61ee73885976f Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 13 Jun 2023 01:07:21 -0700 Subject: [PATCH 3/8] change field Signed-off-by: Ben Ye --- pkg/block/metadata/meta.go | 7 +------ pkg/compact/compact.go | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/block/metadata/meta.go b/pkg/block/metadata/meta.go index 4ddf924d39..89ce68c1a2 100644 --- a/pkg/block/metadata/meta.go +++ b/pkg/block/metadata/meta.go @@ -96,13 +96,8 @@ type Thanos struct { } type IndexStats struct { - SeriesMinSize int64 `json:"series_min_size,omitempty"` SeriesMaxSize int64 `json:"series_max_size,omitempty"` - SeriesAvgSize int64 `json:"series_avg_size,omitempty"` - - ChunkMinSize int64 `json:"chunk_min_size,omitempty"` - ChunkMaxSize int64 `json:"chunk_max_size,omitempty"` - ChunkAvgSize int64 `json:"chunk_avg_size,omitempty"` + ChunkMaxSize int64 `json:"chunk_max_size,omitempty"` } type Rewrite struct { diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index c6393e9891..da95fbaa0b 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -1128,12 +1128,8 @@ func (cg *Group) compact(ctx context.Context, dir string, planner Planner, comp Source: metadata.CompactorSource, SegmentFiles: block.GetSegmentFiles(bdir), IndexStats: metadata.IndexStats{ - SeriesMinSize: stats.SeriesMinSize, SeriesMaxSize: stats.SeriesMaxSize, - SeriesAvgSize: stats.SeriesAvgSize, - ChunkMinSize: stats.ChunkMinSize, ChunkMaxSize: stats.ChunkMaxSize, - ChunkAvgSize: stats.ChunkAvgSize, }, }, nil) if err != nil { From 203cb368ce4f817fdea72d854b2a34745d0e4d35 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 13 Jun 2023 15:58:23 -0700 Subject: [PATCH 4/8] update downsample Signed-off-by: Ben Ye --- cmd/thanos/downsample.go | 19 ++++++++++++++++++- pkg/block/block_test.go | 4 ++-- pkg/block/metadata/meta_test.go | 9 ++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 5b2dbfeca7..a287da0f9a 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -385,10 +385,27 @@ func processDownsampling( "from", m.ULID, "to", id, "duration", downsampleDuration, "duration_ms", downsampleDuration.Milliseconds()) metrics.downsampleDuration.WithLabelValues(m.Thanos.GroupKey()).Observe(downsampleDuration.Seconds()) - if err := block.VerifyIndex(logger, filepath.Join(resdir, block.IndexFilename), m.MinTime, m.MaxTime); err != nil && !acceptMalformedIndex { + stats, err := block.GatherIndexHealthStats(logger, filepath.Join(resdir, block.IndexFilename), m.MinTime, m.MaxTime) + if err == nil { + err = stats.AnyErr() + } + if err != nil && !acceptMalformedIndex { return errors.Wrap(err, "output block index not valid") } + meta, err := metadata.ReadFromDir(resdir) + if err != nil { + return errors.Wrap(err, "read meta") + } + + meta.Thanos.IndexStats = metadata.IndexStats{ + SeriesMaxSize: stats.SeriesMaxSize, + ChunkMaxSize: stats.ChunkMaxSize, + } + if err := meta.WriteToDir(logger, resdir); err != nil { + return errors.Wrap(err, "write meta") + } + begin = time.Now() err = block.Upload(ctx, logger, bkt, resdir, hashFunc) diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 2fb26fc191..b2feb5bce5 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -142,7 +142,7 @@ func TestUpload(t *testing.T) { testutil.Equals(t, 3, len(bkt.Objects())) testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")])) testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b1.String(), IndexFilename)])) - testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)])) + testutil.Equals(t, 567, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)])) // File stats are gathered. testutil.Equals(t, fmt.Sprintf(`{ @@ -192,7 +192,7 @@ func TestUpload(t *testing.T) { testutil.Equals(t, 3, len(bkt.Objects())) testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b1.String(), ChunksDirname, "000001")])) testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b1.String(), IndexFilename)])) - testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)])) + testutil.Equals(t, 567, len(bkt.Objects()[path.Join(b1.String(), MetaFilename)])) } { // Upload with no external labels should be blocked. diff --git a/pkg/block/metadata/meta_test.go b/pkg/block/metadata/meta_test.go index 0be53197e8..6545157349 100644 --- a/pkg/block/metadata/meta_test.go +++ b/pkg/block/metadata/meta_test.go @@ -31,7 +31,8 @@ func TestMeta_ReadWrite(t *testing.T) { "downsample": { "resolution": 0 }, - "source": "" + "source": "", + "index_stats": {} } } `, b.String()) @@ -121,7 +122,8 @@ func TestMeta_ReadWrite(t *testing.T) { { "rel_path": "meta.json" } - ] + ], + "index_stats": {} } } `, b.String()) @@ -199,7 +201,8 @@ func TestMeta_ReadWrite(t *testing.T) { "rel_path": "index", "size_bytes": 1313 } - ] + ], + "index_stats": {} } } `, b.String()) From 3a2b5126b6a3e89b4a245ef4deece62354576bab Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 13 Jun 2023 16:22:51 -0700 Subject: [PATCH 5/8] add index stats into compaction Signed-off-by: Ben Ye --- pkg/block/metadata/meta_test.go | 9 ++++++++- pkg/compact/compact_e2e_test.go | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/block/metadata/meta_test.go b/pkg/block/metadata/meta_test.go index 6545157349..94de50da64 100644 --- a/pkg/block/metadata/meta_test.go +++ b/pkg/block/metadata/meta_test.go @@ -74,6 +74,10 @@ func TestMeta_ReadWrite(t *testing.T) { Downsample: ThanosDownsample{ Resolution: 123144, }, + IndexStats: IndexStats{ + SeriesMaxSize: 2000, + ChunkMaxSize: 1000, + }, }, } testutil.Ok(t, m1.Write(&b)) @@ -123,7 +127,10 @@ func TestMeta_ReadWrite(t *testing.T) { "rel_path": "meta.json" } ], - "index_stats": {} + "index_stats": { + "series_max_size": 2000, + "chunk_max_size": 1000 + } } } `, b.String()) diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index 9b40da16dc..fc279a86fb 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -399,6 +399,8 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg testutil.Assert(t, labels.Equal(extLabels, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) testutil.Assert(t, len(meta.Thanos.SegmentFiles) > 0, "compacted blocks have segment files set") + testutil.Assert(t, meta.Thanos.IndexStats.ChunkMaxSize > 0, "compacted blocks have index stats chunk max size set") + testutil.Assert(t, meta.Thanos.IndexStats.SeriesMaxSize > 0, "compacted blocks have index stats chunk max size set") } { meta, ok := others[groupKey2] @@ -415,6 +417,8 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg testutil.Assert(t, labels.Equal(extLabels2, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) testutil.Assert(t, len(meta.Thanos.SegmentFiles) > 0, "compacted blocks have segment files set") + testutil.Assert(t, meta.Thanos.IndexStats.ChunkMaxSize > 0, "compacted blocks have index stats chunk max size set") + testutil.Assert(t, meta.Thanos.IndexStats.SeriesMaxSize > 0, "compacted blocks have index stats chunk max size set") } }) } From 98a58725f753d6cb4b931be9d0218f184c293666 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 15 Jun 2023 14:11:16 -0700 Subject: [PATCH 6/8] fix tests Signed-off-by: Ben Ye --- cmd/thanos/downsample.go | 8 +++++--- pkg/compact/compact.go | 15 +++++++++------ pkg/compact/compact_e2e_test.go | 8 ++++---- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index a287da0f9a..366eb16145 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -398,9 +398,11 @@ func processDownsampling( return errors.Wrap(err, "read meta") } - meta.Thanos.IndexStats = metadata.IndexStats{ - SeriesMaxSize: stats.SeriesMaxSize, - ChunkMaxSize: stats.ChunkMaxSize, + if stats.ChunkMaxSize > 0 { + meta.Thanos.IndexStats.ChunkMaxSize = stats.ChunkMaxSize + } + if stats.SeriesMaxSize > 0 { + meta.Thanos.IndexStats.SeriesMaxSize = stats.SeriesMaxSize } if err := meta.WriteToDir(logger, resdir); err != nil { return errors.Wrap(err, "write meta") diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go index da95fbaa0b..69aa553713 100644 --- a/pkg/compact/compact.go +++ b/pkg/compact/compact.go @@ -1122,16 +1122,19 @@ func (cg *Group) compact(ctx context.Context, dir string, planner Planner, comp return false, ulid.ULID{}, halt(errors.Wrapf(err, "invalid result block %s", bdir)) } - newMeta, err = metadata.InjectThanos(cg.logger, bdir, metadata.Thanos{ + thanosMeta := metadata.Thanos{ Labels: cg.labels.Map(), Downsample: metadata.ThanosDownsample{Resolution: cg.resolution}, Source: metadata.CompactorSource, SegmentFiles: block.GetSegmentFiles(bdir), - IndexStats: metadata.IndexStats{ - SeriesMaxSize: stats.SeriesMaxSize, - ChunkMaxSize: stats.ChunkMaxSize, - }, - }, nil) + } + if stats.ChunkMaxSize > 0 { + thanosMeta.IndexStats.ChunkMaxSize = stats.ChunkMaxSize + } + if stats.SeriesMaxSize > 0 { + thanosMeta.IndexStats.SeriesMaxSize = stats.SeriesMaxSize + } + newMeta, err = metadata.InjectThanos(cg.logger, bdir, thanosMeta, nil) if err != nil { return false, ulid.ULID{}, errors.Wrapf(err, "failed to finalize the block %s", bdir) } diff --git a/pkg/compact/compact_e2e_test.go b/pkg/compact/compact_e2e_test.go index fc279a86fb..9a798ece61 100644 --- a/pkg/compact/compact_e2e_test.go +++ b/pkg/compact/compact_e2e_test.go @@ -399,8 +399,8 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg testutil.Assert(t, labels.Equal(extLabels, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) testutil.Assert(t, len(meta.Thanos.SegmentFiles) > 0, "compacted blocks have segment files set") - testutil.Assert(t, meta.Thanos.IndexStats.ChunkMaxSize > 0, "compacted blocks have index stats chunk max size set") - testutil.Assert(t, meta.Thanos.IndexStats.SeriesMaxSize > 0, "compacted blocks have index stats chunk max size set") + // Only one chunk will be generated in that block, so we won't set chunk size. + testutil.Assert(t, meta.Thanos.IndexStats.SeriesMaxSize > 0, "compacted blocks have index stats series max size set") } { meta, ok := others[groupKey2] @@ -417,8 +417,8 @@ func testGroupCompactE2e(t *testing.T, mergeFunc storage.VerticalChunkSeriesMerg testutil.Assert(t, labels.Equal(extLabels2, labels.FromMap(meta.Thanos.Labels)), "ext labels does not match") testutil.Equals(t, int64(124), meta.Thanos.Downsample.Resolution) testutil.Assert(t, len(meta.Thanos.SegmentFiles) > 0, "compacted blocks have segment files set") - testutil.Assert(t, meta.Thanos.IndexStats.ChunkMaxSize > 0, "compacted blocks have index stats chunk max size set") - testutil.Assert(t, meta.Thanos.IndexStats.SeriesMaxSize > 0, "compacted blocks have index stats chunk max size set") + // Only one chunk will be generated in that block, so we won't set chunk size. + testutil.Assert(t, meta.Thanos.IndexStats.SeriesMaxSize > 0, "compacted blocks have index stats series max size set") } }) } From c69b5a2d0d324649091ea26bc367281c4820b1a9 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 15 Jun 2023 14:40:28 -0700 Subject: [PATCH 7/8] fix test Signed-off-by: Ben Ye --- pkg/block/block_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index b2feb5bce5..432129e3b1 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -181,7 +181,8 @@ func TestUpload(t *testing.T) { { "rel_path": "meta.json" } - ] + ], + "index_stats": {} } } `, b1.String(), b1.String()), string(bkt.Objects()[path.Join(b1.String(), MetaFilename)])) @@ -224,7 +225,7 @@ func TestUpload(t *testing.T) { testutil.Equals(t, 6, len(bkt.Objects())) testutil.Equals(t, 3727, len(bkt.Objects()[path.Join(b2.String(), ChunksDirname, "000001")])) testutil.Equals(t, 401, len(bkt.Objects()[path.Join(b2.String(), IndexFilename)])) - testutil.Equals(t, 525, len(bkt.Objects()[path.Join(b2.String(), MetaFilename)])) + testutil.Equals(t, 546, len(bkt.Objects()[path.Join(b2.String(), MetaFilename)])) } } From 36071b51e05a178f36b7eef0ef4117a87524c8bc Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 15 Jun 2023 14:43:57 -0700 Subject: [PATCH 8/8] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee22ef45ca..c3166992d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#5777](https://github.com/thanos-io/thanos/pull/5777) Receive: Allow specifying tenant-specific external labels in Router Ingestor. - [#6352](https://github.com/thanos-io/thanos/pull/6352) Store: Expose store gateway query stats in series response hints. - [#6420](https://github.com/thanos-io/thanos/pull/6420) Index Cache: Cache expanded postings. +- [#6441](https://github.com/thanos-io/thanos/pull/6441) Compact: Compactor will set `index_stats` in `meta.json` file with max series and chunk size information. ### Fixed - [#6427](https://github.com/thanos-io/thanos/pull/6427) Receive: increasing log level for failed uploads to error