From 3b2305a06d66682252d4d3c9f4ce25cb0cf42ca8 Mon Sep 17 00:00:00 2001 From: Max Chandler Date: Fri, 29 Jan 2021 14:31:56 +0000 Subject: [PATCH 1/6] add tsdb.allow-overlapping-blocks flag to receiver Signed-off-by: Max Chandler --- cmd/thanos/receive.go | 4 +++- docs/components/receive.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index b90779a008..a4e46992c7 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -87,7 +87,8 @@ func registerReceive(app *extkingpin.App) { tsdbMinBlockDuration := extkingpin.ModelDuration(cmd.Flag("tsdb.min-block-duration", "Min duration for local TSDB blocks").Default("2h").Hidden()) tsdbMaxBlockDuration := extkingpin.ModelDuration(cmd.Flag("tsdb.max-block-duration", "Max duration for local TSDB blocks").Default("2h").Hidden()) - walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() + tsdbAllowOverlappingBlocks := cmd.Flag("tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").Default("false").Bool() + walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() noLockFile := cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup.").Default("false").Bool() ignoreBlockSize := cmd.Flag("shipper.ignore-unequal-block-size", "If true receive will not require min and max block size flags to be set to the same value. Only use this if you want to keep long retention and compaction enabled, as in the worst case it can result in ~2h data loss for your Thanos bucket storage.").Default("false").Hidden().Bool() @@ -113,6 +114,7 @@ func registerReceive(app *extkingpin.App) { RetentionDuration: int64(time.Duration(*retention) / time.Millisecond), NoLockfile: *noLockFile, WALCompression: *walCompression, + AllowOverlappingBlocks: *tsdbAllowOverlappingBlocks, } // Local is empty, so try to generate a local endpoint diff --git a/docs/components/receive.md b/docs/components/receive.md index d937879984..ffe441a02c 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -182,5 +182,7 @@ Flags: --tsdb.no-lockfile Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup. - + --tsdb.allow-overlapping-blocks + Allow overlapping blocks, which in turn enables + vertical compaction and vertical query merge. ``` From 3e53ad11bdab452c4ed505ed80715c509f5474aa Mon Sep 17 00:00:00 2001 From: Max Chandler Date: Fri, 29 Jan 2021 14:43:48 +0000 Subject: [PATCH 2/6] make format Signed-off-by: Max Chandler --- cmd/thanos/receive.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index a4e46992c7..0b63ac211a 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -87,8 +87,8 @@ func registerReceive(app *extkingpin.App) { tsdbMinBlockDuration := extkingpin.ModelDuration(cmd.Flag("tsdb.min-block-duration", "Min duration for local TSDB blocks").Default("2h").Hidden()) tsdbMaxBlockDuration := extkingpin.ModelDuration(cmd.Flag("tsdb.max-block-duration", "Max duration for local TSDB blocks").Default("2h").Hidden()) - tsdbAllowOverlappingBlocks := cmd.Flag("tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").Default("false").Bool() - walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() + tsdbAllowOverlappingBlocks := cmd.Flag("tsdb.allow-overlapping-blocks", "Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge.").Default("false").Bool() + walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool() noLockFile := cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup.").Default("false").Bool() ignoreBlockSize := cmd.Flag("shipper.ignore-unequal-block-size", "If true receive will not require min and max block size flags to be set to the same value. Only use this if you want to keep long retention and compaction enabled, as in the worst case it can result in ~2h data loss for your Thanos bucket storage.").Default("false").Hidden().Bool() @@ -109,12 +109,12 @@ func registerReceive(app *extkingpin.App) { } tsdbOpts := &tsdb.Options{ - MinBlockDuration: int64(time.Duration(*tsdbMinBlockDuration) / time.Millisecond), - MaxBlockDuration: int64(time.Duration(*tsdbMaxBlockDuration) / time.Millisecond), - RetentionDuration: int64(time.Duration(*retention) / time.Millisecond), - NoLockfile: *noLockFile, - WALCompression: *walCompression, - AllowOverlappingBlocks: *tsdbAllowOverlappingBlocks, + MinBlockDuration: int64(time.Duration(*tsdbMinBlockDuration) / time.Millisecond), + MaxBlockDuration: int64(time.Duration(*tsdbMaxBlockDuration) / time.Millisecond), + RetentionDuration: int64(time.Duration(*retention) / time.Millisecond), + NoLockfile: *noLockFile, + WALCompression: *walCompression, + AllowOverlappingBlocks: *tsdbAllowOverlappingBlocks, } // Local is empty, so try to generate a local endpoint From 4d7da2a17cc397c1a43ada73aa9198fb130b8bba Mon Sep 17 00:00:00 2001 From: Matt Lawrence Date: Fri, 29 Jan 2021 15:18:14 +0000 Subject: [PATCH 3/6] cleanup whitespace Signed-off-by: Max Chandler --- docs/components/receive.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/components/receive.md b/docs/components/receive.md index ffe441a02c..5c8ce9d231 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -182,7 +182,7 @@ Flags: --tsdb.no-lockfile Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup. - --tsdb.allow-overlapping-blocks - Allow overlapping blocks, which in turn enables + --tsdb.allow-overlapping-blocks + Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge. ``` From 8ae18f5a8e34dd7ed8ad1254ece70f076acd315e Mon Sep 17 00:00:00 2001 From: Matt Lawrence Date: Fri, 29 Jan 2021 15:21:28 +0000 Subject: [PATCH 4/6] cleanup whitespace Signed-off-by: Max Chandler --- docs/components/receive.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/components/receive.md b/docs/components/receive.md index 5c8ce9d231..ee71577309 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -185,4 +185,5 @@ Flags: --tsdb.allow-overlapping-blocks Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge. + ``` From 53826cface8f732b19836030bcf12ba8f8f1acab Mon Sep 17 00:00:00 2001 From: Max Chandler Date: Fri, 12 Feb 2021 11:34:36 +0000 Subject: [PATCH 5/6] make docs Signed-off-by: Max Chandler --- CHANGELOG.md | 2 +- docs/components/receive.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1d6411be..bac1ccd86d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re - [#3700](https://github.com/thanos-io/thanos/pull/3700) ui: make old bucket viewer UI work with vanilla Prometheus blocks - [#2641](https://github.com/thanos-io/thanos/issues/2641) Query Frontend: Added `--query-range.request-downsampled` flag enabling additional queries for downsampled data in case of empty or incomplete response to range request. - +- [#3792](https://github.com/thanos-io/thanos/pull/3792) Receiver: Added `--tsdb.allow-overlapping-blocks` flag to allow overlapping tsdb blocks and enable vertical compaction ### Changed - [#3705](https://github.com/thanos-io/thanos/pull/3705) Store: Fix race condition leading to failing queries or possibly incorrect query results. diff --git a/docs/components/receive.md b/docs/components/receive.md index ee71577309..28ca10d19d 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -178,12 +178,12 @@ Flags: --receive.replication-factor=1 How many times to replicate incoming write requests. + --tsdb.allow-overlapping-blocks + Allow overlapping blocks, which in turn enables + vertical compaction and vertical query merge. --tsdb.wal-compression Compress the tsdb WAL. --tsdb.no-lockfile Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup. - --tsdb.allow-overlapping-blocks - Allow overlapping blocks, which in turn enables - vertical compaction and vertical query merge. ``` From 14813a35375c5338929a377505181c62815bb042 Mon Sep 17 00:00:00 2001 From: Max Chandler Date: Fri, 12 Feb 2021 12:11:09 +0000 Subject: [PATCH 6/6] fix changelog conflict Signed-off-by: Max Chandler --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bac1ccd86d..460d28ccf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re - [#3700](https://github.com/thanos-io/thanos/pull/3700) ui: make old bucket viewer UI work with vanilla Prometheus blocks - [#2641](https://github.com/thanos-io/thanos/issues/2641) Query Frontend: Added `--query-range.request-downsampled` flag enabling additional queries for downsampled data in case of empty or incomplete response to range request. - [#3792](https://github.com/thanos-io/thanos/pull/3792) Receiver: Added `--tsdb.allow-overlapping-blocks` flag to allow overlapping tsdb blocks and enable vertical compaction + +### Fixed + +- [#3773](https://github.com/thanos-io/thanos/pull/3773) Compact: Pad compaction planner size check + ### Changed - [#3705](https://github.com/thanos-io/thanos/pull/3705) Store: Fix race condition leading to failing queries or possibly incorrect query results.