diff --git a/docs/changelog/94134.yaml b/docs/changelog/94134.yaml new file mode 100644 index 0000000000000..c1e96550cab2a --- /dev/null +++ b/docs/changelog/94134.yaml @@ -0,0 +1,5 @@ +pr: 94134 +summary: Increase the merge factor to 32 for time-based data +area: Engine +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java index 4a8d5ed905cff..02d4f2e5ca6ca 100644 --- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java @@ -103,7 +103,11 @@ public final class MergePolicyConfig { private final TieredMergePolicy tieredMergePolicy = new TieredMergePolicy(); - private final LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy(); + /** + * A merge policy that optimizes for time-based data. It uses Lucene's LogByteSizeMergePolicy, which only merges adjacent segments. In + * turn, this creates segments that have non-overlapping @timestamp ranges if data gets ingested in order. + */ + private final LogByteSizeMergePolicy timeBasedMergePolicy = new LogByteSizeMergePolicy(); private final Logger logger; private final boolean mergesEnabled; private volatile Type mergePolicyType; @@ -120,13 +124,13 @@ public final class MergePolicyConfig { public static final ByteSizeValue DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT = new ByteSizeValue(100, ByteSizeUnit.GB); public static final double DEFAULT_SEGMENTS_PER_TIER = 10.0d; /** - * A default value for {@link LogByteSizeMergePolicy}'s merge factor: 16. This default value differs from the Lucene default of 10 in - * order to account for the fact that Elasticsearch uses {@link LogByteSizeMergePolicy} for time-based data, where it usually makes - * sense to merge data less aggressively, and because {@link LogByteSizeMergePolicy} merges segments more aggressively than - * {@link TieredMergePolicy} for the same number of segments per tier / merge factor because {@link TieredMergePolicy} makes decisions - * at the whole index level, while {@link LogByteSizeMergePolicy} makes decisions on a per-tier basis. + * A default value for {@link LogByteSizeMergePolicy}'s merge factor: 32. This default value differs from the Lucene default of 10 in + * order to account for the fact that Elasticsearch uses {@link LogByteSizeMergePolicy} for time-based data, where adjacent segment + * merging ensures that segments have mostly non-overlapping time ranges if data gets ingested in timestamp order. In turn, this allows + * range queries on the timestamp to remain efficient with high numbers of segments since most segments either don't match the query + * range or are fully contained by the query range. */ - public static final int DEFAULT_MERGE_FACTOR = 16; + public static final int DEFAULT_MERGE_FACTOR = 32; public static final double DEFAULT_DELETES_PCT_ALLOWED = 20.0d; private static final String INDEX_COMPOUND_FORMAT_SETTING_KEY = "index.compound_format"; public static final Setting INDEX_COMPOUND_FORMAT_SETTING = new Setting<>( @@ -158,7 +162,7 @@ MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) { // exponential sizes. The main difference is that it never merges non-adjacent segments, which is an interesting // property for time-based data as described above. - return config.logByteSizeMergePolicy; + return config.timeBasedMergePolicy; } else { return config.tieredMergePolicy; } @@ -170,10 +174,10 @@ MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) { return config.tieredMergePolicy; } }, - LOG_BYTE_SIZE { + TIME_BASED { @Override MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) { - return config.logByteSizeMergePolicy; + return config.timeBasedMergePolicy; } }; @@ -301,17 +305,17 @@ void setSegmentsPerTier(double segmentsPerTier) { void setMergeFactor(int mergeFactor) { // TieredMergePolicy ignores this setting, it configures a number of segments per tier instead, which has different semantics. - logByteSizeMergePolicy.setMergeFactor(mergeFactor); + timeBasedMergePolicy.setMergeFactor(mergeFactor); } void setMaxMergedSegment(ByteSizeValue maxMergedSegment) { // We use 0 as a placeholder for "unset". if (maxMergedSegment.getBytes() == 0) { tieredMergePolicy.setMaxMergedSegmentMB(DEFAULT_MAX_MERGED_SEGMENT.getMbFrac()); - logByteSizeMergePolicy.setMaxMergeMB(DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac()); + timeBasedMergePolicy.setMaxMergeMB(DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac()); } else { tieredMergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac()); - logByteSizeMergePolicy.setMaxMergeMB(maxMergedSegment.getMbFrac()); + timeBasedMergePolicy.setMaxMergeMB(maxMergedSegment.getMbFrac()); } } @@ -322,7 +326,7 @@ void setMaxMergesAtOnce(int maxMergeAtOnce) { void setFloorSegmentSetting(ByteSizeValue floorSegementSetting) { tieredMergePolicy.setFloorSegmentMB(floorSegementSetting.getMbFrac()); - logByteSizeMergePolicy.setMinMergeMB(floorSegementSetting.getMbFrac()); + timeBasedMergePolicy.setMinMergeMB(floorSegementSetting.getMbFrac()); } void setExpungeDeletesAllowed(Double value) { @@ -332,7 +336,7 @@ void setExpungeDeletesAllowed(Double value) { void setCompoundFormatThreshold(CompoundFileThreshold compoundFileThreshold) { compoundFileThreshold.configure(tieredMergePolicy); - compoundFileThreshold.configure(logByteSizeMergePolicy); + compoundFileThreshold.configure(timeBasedMergePolicy); } void setDeletesPctAllowed(Double deletesPctAllowed) { diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java index 8619454bc111e..dbad9dd1cbdb5 100644 --- a/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java +++ b/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java @@ -113,10 +113,7 @@ public void testUpdateSettings() throws IOException { ); assertThat(indexSettings.getMergePolicy(randomBoolean()), Matchers.instanceOf(TieredMergePolicy.class)); indexSettings.updateIndexMetadata( - newIndexMeta( - "index", - Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_TYPE_SETTING.getKey(), "log_byte_size").build() - ) + newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_TYPE_SETTING.getKey(), "time_based").build()) ); assertThat(indexSettings.getMergePolicy(randomBoolean()), Matchers.instanceOf(LogByteSizeMergePolicy.class)); }