Add blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples fl…

…ag (#9981) * Add blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples flag Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Update pkg/storage/tsdb/config.go Co-authored-by: chencs <casie.chen@grafana.com> * Make flag experimental Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> --------- Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> Co-authored-by: chencs <casie.chen@grafana.com>
grafana · Nov 21, 2024 · 23f9fb6 · 23f9fb6
1 parent f7114a1
commit 23f9fb6
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 18 deletions.
diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json
@@ -10187,6 +10187,17 @@
               "fieldType": "int",
               "fieldCategory": "advanced"
             },
+            {
+              "kind": "field",
+              "name": "bigger_out_of_order_blocks_for_old_samples",
+              "required": false,
+              "desc": "When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.",
+              "fieldValue": null,
+              "fieldDefaultValue": false,
+              "fieldFlag": "blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples",
+              "fieldType": "boolean",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "field",
               "name": "series_hash_cache_max_size_bytes",

diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl
@@ -911,6 +911,8 @@ Usage of ./cmd/mimir/mimir:
     	OpenStack Swift user ID.
   -blocks-storage.swift.username string
     	OpenStack Swift username.
+  -blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples
+    	[experimental] When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.
   -blocks-storage.tsdb.block-postings-for-matchers-cache-force
     	[experimental] Force the cache to be used for postings for matchers in compacted blocks, even if it's not a concurrent (query-sharding) call.
   -blocks-storage.tsdb.block-postings-for-matchers-cache-max-bytes int

diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md
@@ -4390,6 +4390,11 @@ tsdb:
   # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
   [head_chunks_write_queue_size: <int> | default = 1000000]
 
+  # (experimental) When enabled, ingester produces 24h blocks for out-of-order
+  # data that is before the current day, instead of the usual 2h blocks.
+  # CLI flag: -blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples
+  [bigger_out_of_order_blocks_for_old_samples: <boolean> | default = false]
+
   # (advanced) Max size - in bytes - of the in-memory series hash cache. The
   # cache is shared across all tenants and it's used only when query sharding is
   # enabled.

diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go
@@ -2687,6 +2687,7 @@ func (i *Ingester) createTSDB(userID string, walReplayConcurrency int) (*userTSD
 		MaxExemplars:                          int64(i.limiter.maxExemplarsPerUser(userID)),
 		SeriesHashCache:                       i.seriesHashCache,
 		EnableMemorySnapshotOnShutdown:        i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown,
+		EnableBiggerOOOBlockForOldSamples:     i.cfg.BlocksStorageConfig.TSDB.BiggerOutOfOrderBlocksForOldSamples,
 		IsolationDisabled:                     true,
 		HeadChunksWriteQueueSize:              i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize,
 		EnableOverlappingCompaction:           false,                // always false since Mimir only uploads lvl 1 compacted blocks

diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go
@@ -205,24 +205,25 @@ func (cfg *BlocksStorageConfig) Validate(activeSeriesCfg activeseries.Config) er
 //
 //nolint:revive
 type TSDBConfig struct {
-	Dir                       string        `yaml:"dir"`
-	BlockRanges               DurationList  `yaml:"block_ranges_period" category:"experimental" doc:"hidden"`
-	Retention                 time.Duration `yaml:"retention_period"`
-	ShipInterval              time.Duration `yaml:"ship_interval" category:"advanced"`
-	ShipConcurrency           int           `yaml:"ship_concurrency" category:"advanced"`
-	HeadCompactionInterval    time.Duration `yaml:"head_compaction_interval" category:"advanced"`
-	HeadCompactionConcurrency int           `yaml:"head_compaction_concurrency" category:"advanced"`
-	HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout" category:"advanced"`
-	HeadChunksWriteBufferSize int           `yaml:"head_chunks_write_buffer_size_bytes" category:"advanced"`
-	HeadChunksEndTimeVariance float64       `yaml:"head_chunks_end_time_variance" category:"experimental"`
-	StripeSize                int           `yaml:"stripe_size" category:"advanced"`
-	WALCompressionEnabled     bool          `yaml:"wal_compression_enabled" category:"advanced"`
-	WALSegmentSizeBytes       int           `yaml:"wal_segment_size_bytes" category:"advanced"`
-	WALReplayConcurrency      int           `yaml:"wal_replay_concurrency" category:"advanced"`
-	FlushBlocksOnShutdown     bool          `yaml:"flush_blocks_on_shutdown" category:"advanced"`
-	CloseIdleTSDBTimeout      time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"`
-	MemorySnapshotOnShutdown  bool          `yaml:"memory_snapshot_on_shutdown" category:"experimental"`
-	HeadChunksWriteQueueSize  int           `yaml:"head_chunks_write_queue_size" category:"advanced"`
+	Dir                                 string        `yaml:"dir"`
+	BlockRanges                         DurationList  `yaml:"block_ranges_period" category:"experimental" doc:"hidden"`
+	Retention                           time.Duration `yaml:"retention_period"`
+	ShipInterval                        time.Duration `yaml:"ship_interval" category:"advanced"`
+	ShipConcurrency                     int           `yaml:"ship_concurrency" category:"advanced"`
+	HeadCompactionInterval              time.Duration `yaml:"head_compaction_interval" category:"advanced"`
+	HeadCompactionConcurrency           int           `yaml:"head_compaction_concurrency" category:"advanced"`
+	HeadCompactionIdleTimeout           time.Duration `yaml:"head_compaction_idle_timeout" category:"advanced"`
+	HeadChunksWriteBufferSize           int           `yaml:"head_chunks_write_buffer_size_bytes" category:"advanced"`
+	HeadChunksEndTimeVariance           float64       `yaml:"head_chunks_end_time_variance" category:"experimental"`
+	StripeSize                          int           `yaml:"stripe_size" category:"advanced"`
+	WALCompressionEnabled               bool          `yaml:"wal_compression_enabled" category:"advanced"`
+	WALSegmentSizeBytes                 int           `yaml:"wal_segment_size_bytes" category:"advanced"`
+	WALReplayConcurrency                int           `yaml:"wal_replay_concurrency" category:"advanced"`
+	FlushBlocksOnShutdown               bool          `yaml:"flush_blocks_on_shutdown" category:"advanced"`
+	CloseIdleTSDBTimeout                time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"`
+	MemorySnapshotOnShutdown            bool          `yaml:"memory_snapshot_on_shutdown" category:"experimental"`
+	HeadChunksWriteQueueSize            int           `yaml:"head_chunks_write_queue_size" category:"advanced"`
+	BiggerOutOfOrderBlocksForOldSamples bool          `yaml:"bigger_out_of_order_blocks_for_old_samples" category:"experimental"`
 
 	// Series hash cache.
 	SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"`
@@ -327,6 +328,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) {
 	f.Int64Var(&cfg.EarlyHeadCompactionMinInMemorySeries, "blocks-storage.tsdb.early-head-compaction-min-in-memory-series", 0, fmt.Sprintf("When the number of in-memory series in the ingester is equal to or greater than this setting, the ingester tries to compact the TSDB Head. The early compaction removes from the memory all samples and inactive series up until -%s time ago. After an early compaction, the ingester will not accept any sample with a timestamp older than -%s time ago (unless out of order ingestion is enabled). The ingester checks every -%s whether an early compaction is required. Use 0 to disable it.", activeseries.IdleTimeoutFlag, activeseries.IdleTimeoutFlag, headCompactionIntervalFlag))
 	f.IntVar(&cfg.EarlyHeadCompactionMinEstimatedSeriesReductionPercentage, "blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage", 15, "When the early compaction is enabled, the early compaction is triggered only if the estimated series reduction is at least the configured percentage (0-100).")
 	f.BoolVar(&cfg.TimelyHeadCompaction, "blocks-storage.tsdb.timely-head-compaction-enabled", false, "Allows head compaction to happen when the min block range can no longer be appended, without requiring 1.5x the chunk range worth of data in the head.")
+	f.BoolVar(&cfg.BiggerOutOfOrderBlocksForOldSamples, "blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples", false, "When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.")
 
 	cfg.HeadCompactionIntervalJitterEnabled = true
 	cfg.HeadCompactionIntervalWhileStarting = 30 * time.Second