[1.5] Adjustable TSI Compaction Threshold

etc/config.sample.toml

@@ -97,6 +97,15 @@
   # to cache snapshotting.
   # max-concurrent-compactions = 0
 
+  # The threshold, in bytes, when an index write-ahead log file will compact
+  # into an index file. Lower sizes will cause log files to be compacted more
+  # quickly and result in lower heap usage at the expense of write throughput.
+  # Higher sizes will be compacted less frequently, store more series in-memory,
+  # and provide higher write throughput.
+  # Valid size suffixes are k, m, or g (case insensitive, 1024 = 1k).
+  # Values without a size suffix are in bytes.
+  # max-index-log-file-size = "1m"
+
   # The maximum series allowed per database before writes are dropped.  This limit can prevent
   # high cardinality issues at the database level.  This limit can be disabled by setting it to
   # 0.

tsdb/config.go

@@ -51,6 +51,10 @@ const (
 	// DefaultMaxConcurrentCompactions is the maximum number of concurrent full and level compactions
 	// that can run at one time.  A value of 0 results in 50% of runtime.GOMAXPROCS(0) used at runtime.
 	DefaultMaxConcurrentCompactions = 0
+
+	// DefaultMaxIndexLogFileSize is the default threshold, in bytes, when an index
+	// write-ahead log file will compact into an index file.
+	DefaultMaxIndexLogFileSize = 1 * 1024 * 1024 // 1MB
 )
 
 // Config holds the configuration for the tsbd package.
@@ -94,6 +98,12 @@ type Config struct {
 	// not affected by this limit.  A value of 0 limits compactions to runtime.GOMAXPROCS(0).
 	MaxConcurrentCompactions int `toml:"max-concurrent-compactions"`
 
+	// MaxIndexLogFileSize is the threshold, in bytes, when an index write-ahead log file will
+	// compact into an index file. Lower sizes will cause log files to be compacted more quickly
+	// and result in lower heap usage at the expense of write throughput. Higher sizes will
+	// be compacted less frequently, store more series in-memory, and provide higher write throughput.
+	MaxIndexLogFileSize toml.Size `toml:"max-index-log-file-size"`
+
 	TraceLoggingEnabled bool `toml:"trace-logging-enabled"`
 }
 
@@ -114,6 +124,8 @@ func NewConfig() Config {
 		MaxValuesPerTag:          DefaultMaxValuesPerTag,
 		MaxConcurrentCompactions: DefaultMaxConcurrentCompactions,
 
+		MaxIndexLogFileSize: toml.Size(DefaultMaxIndexLogFileSize),
+
 		TraceLoggingEnabled: false,
 	}
 }

tsdb/index/tsi1/index.go

@@ -39,8 +39,8 @@ func init() {
 		DefaultPartitionN = uint64(i)
 	}
 
-	tsdb.RegisterIndex(IndexName, func(_ uint64, db, path string, _ *tsdb.SeriesIDSet, sfile *tsdb.SeriesFile, _ tsdb.EngineOptions) tsdb.Index {
-		idx := NewIndex(sfile, db, WithPath(path))
+	tsdb.RegisterIndex(IndexName, func(_ uint64, db, path string, _ *tsdb.SeriesIDSet, sfile *tsdb.SeriesFile, opt tsdb.EngineOptions) tsdb.Index {
+		idx := NewIndex(sfile, db, WithPath(path), WithMaximumLogFileSize(int64(opt.Config.MaxIndexLogFileSize)))
 		return idx
 	})
 }
@@ -111,7 +111,7 @@ type Index struct {
 // NewIndex returns a new instance of Index.
 func NewIndex(sfile *tsdb.SeriesFile, database string, options ...IndexOption) *Index {
 	idx := &Index{
-		maxLogFileSize: DefaultMaxLogFileSize,
+		maxLogFileSize: tsdb.DefaultMaxIndexLogFileSize,
 		logger:         zap.NewNop(),
 		version:        Version,
 		sfile:          sfile,

tsdb/index/tsi1/partition.go

@@ -25,9 +25,6 @@ import (
 // Version is the current version of the TSI index.
 const Version = 1
 
-// DefaultMaxLogFileSize is the default compaction threshold.
-const DefaultMaxLogFileSize = 5 * 1024 * 1024
-
 // File extensions.
 const (
 	LogFileExt   = ".tsl"
@@ -97,7 +94,7 @@ func NewPartition(sfile *tsdb.SeriesFile, path string) *Partition {
 		seriesIDSet: tsdb.NewSeriesIDSet(),
 
 		// Default compaction thresholds.
-		MaxLogFileSize: DefaultMaxLogFileSize,
+		MaxLogFileSize: tsdb.DefaultMaxIndexLogFileSize,
 
 		// compactionEnabled: true,
 		compactionInterrupt: make(chan struct{}),