From f32f806aa2903a84dad71516abf3085fffaaf4a3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 23 Dec 2021 12:14:17 +0000 Subject: [PATCH 1/2] Disable block deletion marks migration by default Flag is named `-compactor.block-deletion-marks-migration-enabled`. This feature was added in v1.7, so we expect most users to have upgraded by now. Signed-off-by: Bryan Boreham --- CHANGELOG.md | 1 + docs/blocks-storage/production-tips.md | 9 --------- pkg/compactor/compactor.go | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0314fb49dd..9fdff8895c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539 * [CHANGE] query-frontend: Do not print anything in the logs of `query-frontend` if a in-progress query has been canceled (context canceled). #4562 +* [CHANGE] Compactor block deletion mark migration, needed when upgrading from v1.7, is now disabled by default. #4597 * [ENHANCEMENT] Ruler: Add `-ruler.disable-rule-group-label` to disable the `rule_group` label on exported metrics. #4571 * [ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. #4502 * [ENHANCEMENT] Added new ring related config `-ingester.readiness-check-ring-health` when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. #4539 diff --git a/docs/blocks-storage/production-tips.md b/docs/blocks-storage/production-tips.md index 718b2d5bc3..ff400e4fd9 100644 --- a/docs/blocks-storage/production-tips.md +++ b/docs/blocks-storage/production-tips.md @@ -89,15 +89,6 @@ The rule of thumb is that a production system shouldn't have the `file-max` ulim The compactor generally needs a lot of disk space in order to download source blocks from the bucket and store the compacted block before uploading it to the storage. Please refer to [Compactor disk utilization](./compactor.md#compactor-disk-utilization) for more information about how to do capacity planning. -### Ensure deletion marks migration is disabled after first run - -Cortex 1.7.0 introduced a change to store block deletion marks in a per-tenant global `markers/` location in the object store. If your Cortex cluster has been created with a Cortex version older than 1.7.0, the compactor needs to migrate deletion marks to the global location. - -The migration is a one-time operation run at compactor startup. Running it at every compactor startup is a waste of resources and increases the compactor startup time so, once it successfully run once, you should disable it via `-compactor.block-deletion-marks-migration-enabled=false` (or its respective YAML config option). - -You can see that the initial migration is done by looking for the following message in the compactor's logs: -`msg="migrated block deletion marks to the global markers location"`. - ## Caching ### Ensure memcached is properly scaled diff --git a/pkg/compactor/compactor.go b/pkg/compactor/compactor.go index 8ba7b7933d..6bd4d75a60 100644 --- a/pkg/compactor/compactor.go +++ b/pkg/compactor/compactor.go @@ -147,7 +147,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { "If not 0, blocks will be marked for deletion and compactor component will permanently delete blocks marked for deletion from the bucket. "+ "If 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures.") f.DurationVar(&cfg.TenantCleanupDelay, "compactor.tenant-cleanup-delay", 6*time.Hour, "For tenants marked for deletion, this is time between deleting of last block, and doing final cleanup (marker files, debug files) of the tenant.") - f.BoolVar(&cfg.BlockDeletionMarksMigrationEnabled, "compactor.block-deletion-marks-migration-enabled", true, "When enabled, at compactor startup the bucket will be scanned and all found deletion marks inside the block location will be copied to the markers global location too. This option can (and should) be safely disabled as soon as the compactor has successfully run at least once.") + f.BoolVar(&cfg.BlockDeletionMarksMigrationEnabled, "compactor.block-deletion-marks-migration-enabled", false, "When enabled, at compactor startup the bucket will be scanned and all found deletion marks inside the block location will be copied to the markers global location too. This option can (and should) be safely disabled as soon as the compactor has successfully run at least once.") f.Var(&cfg.EnabledTenants, "compactor.enabled-tenants", "Comma separated list of tenants that can be compacted. If specified, only these tenants will be compacted by compactor, otherwise all tenants can be compacted. Subject to sharding.") f.Var(&cfg.DisabledTenants, "compactor.disabled-tenants", "Comma separated list of tenants that cannot be compacted by this compactor. If specified, and compactor would normally pick given tenant for compaction (via -compactor.enabled-tenants or sharding), it will be ignored instead.") From 33d558a699fc4c9d8264da2fbcbc42687a5efb2d Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 23 Dec 2021 13:54:17 +0000 Subject: [PATCH 2/2] make docs Signed-off-by: Bryan Boreham --- docs/blocks-storage/compactor.md | 2 +- docs/configuration/config-file-reference.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/blocks-storage/compactor.md b/docs/blocks-storage/compactor.md index ca30832d93..ada0f9f41a 100644 --- a/docs/blocks-storage/compactor.md +++ b/docs/blocks-storage/compactor.md @@ -152,7 +152,7 @@ compactor: # global location too. This option can (and should) be safely disabled as soon # as the compactor has successfully run at least once. # CLI flag: -compactor.block-deletion-marks-migration-enabled - [block_deletion_marks_migration_enabled: | default = true] + [block_deletion_marks_migration_enabled: | default = false] # Comma separated list of tenants that can be compacted. If specified, only # these tenants will be compacted by compactor, otherwise all tenants can be diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 047fcaba2c..a26fce199c 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5296,7 +5296,7 @@ The `compactor_config` configures the compactor for the blocks storage. # location too. This option can (and should) be safely disabled as soon as the # compactor has successfully run at least once. # CLI flag: -compactor.block-deletion-marks-migration-enabled -[block_deletion_marks_migration_enabled: | default = true] +[block_deletion_marks_migration_enabled: | default = false] # Comma separated list of tenants that can be compacted. If specified, only # these tenants will be compacted by compactor, otherwise all tenants can be