diff --git a/CHANGELOG.md b/CHANGELOG.md index 54f2b2ea040..c9a2f104eef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,9 @@ * [ENHANCEMENT] Distributor: Initialize ha_tracker cache before ha_tracker and distributor reach running state and begin serving writes. #9826 * [ENHANCEMENT] Ingester: `-ingest-storage.kafka.max-buffered-bytes` to limit the memory for buffered records when using concurrent fetching. #9892 * [ENHANCEMENT] Querier: improve performance and memory consumption of queries that select many series. #9914 +* [ENHANCEMENT] Ruler: Support OAuth2 and proxies in Alertmanager client #9945 +* [ENHANCEMENT] Ingester: Add `-blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples` to build 24h blocks for out-of-order data belonging to the previous days instead of building smaller 2h blocks. This reduces pressure on compactors and ingesters when the out-of-order samples span multiple days in the past. #9844 #10033 #10035 +* [ENHANCEMENT] Distributor: allow a different limit for info series (series ending in `_info`) label count, via `-validation.max-label-names-per-info-series`. #10028 * [BUGFIX] Fix issue where functions such as `rate()` over native histograms could return incorrect values if a float stale marker was present in the selected range. #9508 * [BUGFIX] Fix issue where negation of native histograms (eg. `-some_native_histogram_series`) did nothing. #9508 * [BUGFIX] Fix issue where `metric might not be a counter, name does not end in _total/_sum/_count/_bucket` annotation would be emitted even if `rate` or `increase` did not have enough samples to compute a result. #9508 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 8aa0f702ec2..1e49f80abe1 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -10199,6 +10199,17 @@ "fieldType": "int", "fieldCategory": "advanced" }, + { + "kind": "field", + "name": "bigger_out_of_order_blocks_for_old_samples", + "required": false, + "desc": "When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "field", "name": "series_hash_cache_max_size_bytes", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 1e19f1db6f0..3964b93563e 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -911,6 +911,8 @@ Usage of ./cmd/mimir/mimir: OpenStack Swift user ID. -blocks-storage.swift.username string OpenStack Swift username. + -blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples + [experimental] When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks. -blocks-storage.tsdb.block-postings-for-matchers-cache-force [experimental] Force the cache to be used for postings for matchers in compacted blocks, even if it's not a concurrent (query-sharding) call. -blocks-storage.tsdb.block-postings-for-matchers-cache-max-bytes int diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index e889cb32192..75b3c02caf4 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -4395,6 +4395,11 @@ tsdb: # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 1000000] + # (experimental) When enabled, ingester produces 24h blocks for out-of-order + # data that is before the current day, instead of the usual 2h blocks. + # CLI flag: -blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples + [bigger_out_of_order_blocks_for_old_samples: | default = false] + # (advanced) Max size - in bytes - of the in-memory series hash cache. The # cache is shared across all tenants and it's used only when query sharding is # enabled. diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 4daf0751ae8..f844f009143 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -2687,6 +2687,7 @@ func (i *Ingester) createTSDB(userID string, walReplayConcurrency int) (*userTSD MaxExemplars: int64(i.limiter.maxExemplarsPerUser(userID)), SeriesHashCache: i.seriesHashCache, EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown, + EnableBiggerOOOBlockForOldSamples: i.cfg.BlocksStorageConfig.TSDB.BiggerOutOfOrderBlocksForOldSamples, IsolationDisabled: true, HeadChunksWriteQueueSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize, EnableOverlappingCompaction: false, // always false since Mimir only uploads lvl 1 compacted blocks diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 025c0eec51f..6c7dfd2d573 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -205,24 +205,25 @@ func (cfg *BlocksStorageConfig) Validate(activeSeriesCfg activeseries.Config) er // //nolint:revive type TSDBConfig struct { - Dir string `yaml:"dir"` - BlockRanges DurationList `yaml:"block_ranges_period" category:"experimental" doc:"hidden"` - Retention time.Duration `yaml:"retention_period"` - ShipInterval time.Duration `yaml:"ship_interval" category:"advanced"` - ShipConcurrency int `yaml:"ship_concurrency" category:"advanced"` - HeadCompactionInterval time.Duration `yaml:"head_compaction_interval" category:"advanced"` - HeadCompactionConcurrency int `yaml:"head_compaction_concurrency" category:"advanced"` - HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout" category:"advanced"` - HeadChunksWriteBufferSize int `yaml:"head_chunks_write_buffer_size_bytes" category:"advanced"` - HeadChunksEndTimeVariance float64 `yaml:"head_chunks_end_time_variance" category:"experimental"` - StripeSize int `yaml:"stripe_size" category:"advanced"` - WALCompressionEnabled bool `yaml:"wal_compression_enabled" category:"advanced"` - WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes" category:"advanced"` - WALReplayConcurrency int `yaml:"wal_replay_concurrency" category:"advanced"` - FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown" category:"advanced"` - CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"` - MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown" category:"experimental"` - HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size" category:"advanced"` + Dir string `yaml:"dir"` + BlockRanges DurationList `yaml:"block_ranges_period" category:"experimental" doc:"hidden"` + Retention time.Duration `yaml:"retention_period"` + ShipInterval time.Duration `yaml:"ship_interval" category:"advanced"` + ShipConcurrency int `yaml:"ship_concurrency" category:"advanced"` + HeadCompactionInterval time.Duration `yaml:"head_compaction_interval" category:"advanced"` + HeadCompactionConcurrency int `yaml:"head_compaction_concurrency" category:"advanced"` + HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout" category:"advanced"` + HeadChunksWriteBufferSize int `yaml:"head_chunks_write_buffer_size_bytes" category:"advanced"` + HeadChunksEndTimeVariance float64 `yaml:"head_chunks_end_time_variance" category:"experimental"` + StripeSize int `yaml:"stripe_size" category:"advanced"` + WALCompressionEnabled bool `yaml:"wal_compression_enabled" category:"advanced"` + WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes" category:"advanced"` + WALReplayConcurrency int `yaml:"wal_replay_concurrency" category:"advanced"` + FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown" category:"advanced"` + CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"` + MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown" category:"experimental"` + HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size" category:"advanced"` + BiggerOutOfOrderBlocksForOldSamples bool `yaml:"bigger_out_of_order_blocks_for_old_samples" category:"experimental"` // Series hash cache. SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"` @@ -327,6 +328,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.Int64Var(&cfg.EarlyHeadCompactionMinInMemorySeries, "blocks-storage.tsdb.early-head-compaction-min-in-memory-series", 0, fmt.Sprintf("When the number of in-memory series in the ingester is equal to or greater than this setting, the ingester tries to compact the TSDB Head. The early compaction removes from the memory all samples and inactive series up until -%s time ago. After an early compaction, the ingester will not accept any sample with a timestamp older than -%s time ago (unless out of order ingestion is enabled). The ingester checks every -%s whether an early compaction is required. Use 0 to disable it.", activeseries.IdleTimeoutFlag, activeseries.IdleTimeoutFlag, headCompactionIntervalFlag)) f.IntVar(&cfg.EarlyHeadCompactionMinEstimatedSeriesReductionPercentage, "blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage", 15, "When the early compaction is enabled, the early compaction is triggered only if the estimated series reduction is at least the configured percentage (0-100).") f.BoolVar(&cfg.TimelyHeadCompaction, "blocks-storage.tsdb.timely-head-compaction-enabled", false, "Allows head compaction to happen when the min block range can no longer be appended, without requiring 1.5x the chunk range worth of data in the head.") + f.BoolVar(&cfg.BiggerOutOfOrderBlocksForOldSamples, "blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples", false, "When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.") cfg.HeadCompactionIntervalJitterEnabled = true cfg.HeadCompactionIntervalWhileStarting = 30 * time.Second