Increase the merge factor to 32 for time-based data. (#94134)

This is a follow-up to #92684. #92684 switched from `TieredMergePolicy`
to `LogByteSizeMergePolicy` for time-based data, trying to retain
similar numbers of segments in shards. This change goes further, and
takes advantage of the fact that adjacent segment merging gives segments
(mostly) non-overlapping time ranges, to reduce merging overhead without
hurting the efficiency of range queries on the timestamp field.

In general the trade-off of this change is that it yields:  - Faster
ingestion thanks to reduced merging overhead.  - Similar performance for
range queries on the timestamp field.  - Very slightly degraded
performance of term queries due to the increased number of segments.
This should be hardly noticeable in most cases.  - Possibly degraded
performance of fuzzy, wildcard queries, as well as range queries on
other fields than the timestamp field.

docs/changelog/94134.yaml

@@ -0,0 +1,5 @@
+pr: 94134
+summary: Increase the merge factor to 32 for time-based data
+area: Engine
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java

@@ -103,7 +103,11 @@
 
 public final class MergePolicyConfig {
     private final TieredMergePolicy tieredMergePolicy = new TieredMergePolicy();
-    private final LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy();
+    /**
+     * A merge policy that optimizes for time-based data. It uses Lucene's LogByteSizeMergePolicy, which only merges adjacent segments. In
+     * turn, this creates segments that have non-overlapping @timestamp ranges if data gets ingested in order.
+     */
+    private final LogByteSizeMergePolicy timeBasedMergePolicy = new LogByteSizeMergePolicy();
     private final Logger logger;
     private final boolean mergesEnabled;
     private volatile Type mergePolicyType;
@@ -120,13 +124,13 @@ public final class MergePolicyConfig {
     public static final ByteSizeValue DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT = new ByteSizeValue(100, ByteSizeUnit.GB);
     public static final double DEFAULT_SEGMENTS_PER_TIER = 10.0d;
     /**
-     * A default value for {@link LogByteSizeMergePolicy}'s merge factor: 16. This default value differs from the Lucene default of 10 in
-     * order to account for the fact that Elasticsearch uses {@link LogByteSizeMergePolicy} for time-based data, where it usually makes
-     * sense to merge data less aggressively, and because {@link LogByteSizeMergePolicy} merges segments more aggressively than
-     * {@link TieredMergePolicy} for the same number of segments per tier / merge factor because {@link TieredMergePolicy} makes decisions
-     * at the whole index level, while {@link LogByteSizeMergePolicy} makes decisions on a per-tier basis.
+     * A default value for {@link LogByteSizeMergePolicy}'s merge factor: 32. This default value differs from the Lucene default of 10 in
+     * order to account for the fact that Elasticsearch uses {@link LogByteSizeMergePolicy} for time-based data, where adjacent segment
+     * merging ensures that segments have mostly non-overlapping time ranges if data gets ingested in timestamp order. In turn, this allows
+     * range queries on the timestamp to remain efficient with high numbers of segments since most segments either don't match the query
+     * range or are fully contained by the query range.
      */
-    public static final int DEFAULT_MERGE_FACTOR = 16;
+    public static final int DEFAULT_MERGE_FACTOR = 32;
     public static final double DEFAULT_DELETES_PCT_ALLOWED = 20.0d;
     private static final String INDEX_COMPOUND_FORMAT_SETTING_KEY = "index.compound_format";
     public static final Setting<CompoundFileThreshold> INDEX_COMPOUND_FORMAT_SETTING = new Setting<>(
@@ -158,7 +162,7 @@ MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) {
                     // exponential sizes. The main difference is that it never merges non-adjacent segments, which is an interesting
                     // property for time-based data as described above.
 
-                    return config.logByteSizeMergePolicy;
+                    return config.timeBasedMergePolicy;
                 } else {
                     return config.tieredMergePolicy;
                 }
@@ -170,10 +174,10 @@ MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) {
                 return config.tieredMergePolicy;
             }
         },
-        LOG_BYTE_SIZE {
+        TIME_BASED {
             @Override
             MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) {
-                return config.logByteSizeMergePolicy;
+                return config.timeBasedMergePolicy;
             }
         };
 
@@ -301,17 +305,17 @@ void setSegmentsPerTier(double segmentsPerTier) {
 
     void setMergeFactor(int mergeFactor) {
         // TieredMergePolicy ignores this setting, it configures a number of segments per tier instead, which has different semantics.
-        logByteSizeMergePolicy.setMergeFactor(mergeFactor);
+        timeBasedMergePolicy.setMergeFactor(mergeFactor);
     }
 
     void setMaxMergedSegment(ByteSizeValue maxMergedSegment) {
         // We use 0 as a placeholder for "unset".
         if (maxMergedSegment.getBytes() == 0) {
             tieredMergePolicy.setMaxMergedSegmentMB(DEFAULT_MAX_MERGED_SEGMENT.getMbFrac());
-            logByteSizeMergePolicy.setMaxMergeMB(DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac());
+            timeBasedMergePolicy.setMaxMergeMB(DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac());
         } else {
             tieredMergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
-            logByteSizeMergePolicy.setMaxMergeMB(maxMergedSegment.getMbFrac());
+            timeBasedMergePolicy.setMaxMergeMB(maxMergedSegment.getMbFrac());
         }
     }
 
@@ -322,7 +326,7 @@ void setMaxMergesAtOnce(int maxMergeAtOnce) {
 
     void setFloorSegmentSetting(ByteSizeValue floorSegementSetting) {
         tieredMergePolicy.setFloorSegmentMB(floorSegementSetting.getMbFrac());
-        logByteSizeMergePolicy.setMinMergeMB(floorSegementSetting.getMbFrac());
+        timeBasedMergePolicy.setMinMergeMB(floorSegementSetting.getMbFrac());
     }
 
     void setExpungeDeletesAllowed(Double value) {
@@ -332,7 +336,7 @@ void setExpungeDeletesAllowed(Double value) {
 
     void setCompoundFormatThreshold(CompoundFileThreshold compoundFileThreshold) {
         compoundFileThreshold.configure(tieredMergePolicy);
-        compoundFileThreshold.configure(logByteSizeMergePolicy);
+        compoundFileThreshold.configure(timeBasedMergePolicy);
     }
 
     void setDeletesPctAllowed(Double deletesPctAllowed) {

server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java

@@ -113,10 +113,7 @@ public void testUpdateSettings() throws IOException {
         );
         assertThat(indexSettings.getMergePolicy(randomBoolean()), Matchers.instanceOf(TieredMergePolicy.class));
         indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_TYPE_SETTING.getKey(), "log_byte_size").build()
-            )
+            newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_TYPE_SETTING.getKey(), "time_based").build())
         );
         assertThat(indexSettings.getMergePolicy(randomBoolean()), Matchers.instanceOf(LogByteSizeMergePolicy.class));
     }