From 9dd8f6b5b80c6187f360e212dec47d961dc51706 Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Thu, 26 Sep 2024 15:10:22 -0400 Subject: [PATCH] copyblocks: add option to bypass duration check for no-compact blocks --- CHANGELOG.md | 2 ++ tools/copyblocks/README.md | 2 +- tools/copyblocks/main.go | 27 +++++++++++++++------------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 931d70d47b2..974b785953b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ ### Tools +* [ENHANCEMENT] `copyblocks`: Added `--any-no-compact-block-duration`, which defaults to `false`, to simplify targeting blocks that are not awaiting compaction. #9439 + ## v2.14.0-rc.0 ### Grafana Mimir diff --git a/tools/copyblocks/README.md b/tools/copyblocks/README.md index a4fe54a0905..d15ad75c05d 100644 --- a/tools/copyblocks/README.md +++ b/tools/copyblocks/README.md @@ -11,7 +11,7 @@ The currently supported services are Amazon Simple Storage Service (S3 and S3-co - Prevents copying blocks multiple times to the same destination bucket by uploading block marker files to the source bucket - Runs continuously with periodic checks when supplied a time duration with `--copy-period`, otherwise runs one check then exits - Include or exclude users from having blocks copied (`--enabled-users` and `--disabled-users`) -- Configurable minimum block duration (`--min-block-duration`) to avoid copying blocks that will be compacted +- Configurable minimum block duration (`--min-block-duration`) and (`--any-no-compact-block-duration`) to target blocks that are not awaiting compaction - Configurable time range (`--min-time` and `--max-time`) to only copy blocks inclusively within a provided range - Log what would be copied without actually copying anything with `--dry-run` diff --git a/tools/copyblocks/main.go b/tools/copyblocks/main.go index c2f6b8da68b..96807c70faa 100644 --- a/tools/copyblocks/main.go +++ b/tools/copyblocks/main.go @@ -35,17 +35,18 @@ import ( ) type config struct { - copyConfig objtools.CopyBucketConfig - minBlockDuration time.Duration - minTime flagext.Time - maxTime flagext.Time - tenantConcurrency int - blockConcurrency int - copyPeriod time.Duration - enabledUsers flagext.StringSliceCSV - disabledUsers flagext.StringSliceCSV - dryRun bool - httpListen string + copyConfig objtools.CopyBucketConfig + minBlockDuration time.Duration + minTime flagext.Time + maxTime flagext.Time + tenantConcurrency int + blockConcurrency int + copyPeriod time.Duration + enabledUsers flagext.StringSliceCSV + disabledUsers flagext.StringSliceCSV + dryRun bool + anyNoCompactBlockDuration bool + httpListen string } func (c *config) registerFlags(f *flag.FlagSet) { @@ -59,6 +60,7 @@ func (c *config) registerFlags(f *flag.FlagSet) { f.Var(&c.enabledUsers, "enabled-users", "If not empty, only blocks for these users are copied.") f.Var(&c.disabledUsers, "disabled-users", "If not empty, blocks for these users are not copied.") f.BoolVar(&c.dryRun, "dry-run", false, "Don't perform any copy; only log what would happen.") + f.BoolVar(&c.anyNoCompactBlockDuration, "any-no-compact-block-duration", false, "If set, blocks marked as no-compact are not checked against min-block-duration") f.StringVar(&c.httpListen, "http-listen-address", ":8080", "HTTP listen address.") } @@ -269,7 +271,8 @@ func copyBlocks(ctx context.Context, cfg config, logger log.Logger, m *metrics) return nil } - if cfg.minBlockDuration > 0 { + skipDurationCheck := cfg.anyNoCompactBlockDuration && markers[blockID].noCompact + if !skipDurationCheck && cfg.minBlockDuration > 0 { blockDuration := time.Millisecond * time.Duration(blockMeta.MaxTime-blockMeta.MinTime) if blockDuration < cfg.minBlockDuration { level.Debug(logger).Log("msg", "skipping block, block duration is less than the configured minimum duration", "block_duration", blockDuration, "configured_min_duration", cfg.minBlockDuration)