Skip to content

Commit f7ba14d

Browse files
jpountzjpountz
authored
Make sure _forcemerge respects max_num_segments. (#32291)
An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976) will make TieredMergePolicy respect the maximum merged segment size all the time, meaning it will possibly not respect the `max_num_segments` parameter anymore if the shard is larger than the maximum segment size. This change makes sure that `max_num_segments` is respected for now in order to give us time to think about how to integrate this change, and also to delay it until 7.0 as this might be a big-enough change for us to wait for a new major version.
1 parent 48885d2 commit f7ba14d

File tree

4 files changed

+222
-23
lines changed

4 files changed

+222
-23
lines changed

‎server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.index;
21+
22+
import org.apache.lucene.index.FilterMergePolicy;
23+
import org.apache.lucene.index.SegmentCommitInfo;
24+
import org.apache.lucene.index.SegmentInfos;
25+
import org.apache.lucene.index.TieredMergePolicy;
26+
27+
import java.io.IOException;
28+
import java.util.Map;
29+
30+
/**
31+
* Wrapper around {@link TieredMergePolicy} which doesn't respect
32+
* {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges.
33+
* See https://issues.apache.org/jira/browse/LUCENE-7976.
34+
*/
35+
final class EsTieredMergePolicy extends FilterMergePolicy {
36+
37+
final TieredMergePolicy regularMergePolicy;
38+
final TieredMergePolicy forcedMergePolicy;
39+
40+
EsTieredMergePolicy() {
41+
super(new TieredMergePolicy());
42+
regularMergePolicy = (TieredMergePolicy) in;
43+
forcedMergePolicy = new TieredMergePolicy();
44+
forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited
45+
}
46+
47+
@Override
48+
public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount,
49+
Map<SegmentCommitInfo, Boolean> segmentsToMerge, MergeContext mergeContext) throws IOException {
50+
return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext);
51+
}
52+
53+
@Override
54+
public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
55+
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
56+
}
57+
58+
public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
59+
regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
60+
forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
61+
}
62+
63+
public double getForceMergeDeletesPctAllowed() {
64+
return forcedMergePolicy.getForceMergeDeletesPctAllowed();
65+
}
66+
67+
public void setFloorSegmentMB(double mbFrac) {
68+
regularMergePolicy.setFloorSegmentMB(mbFrac);
69+
forcedMergePolicy.setFloorSegmentMB(mbFrac);
70+
}
71+
72+
public double getFloorSegmentMB() {
73+
return regularMergePolicy.getFloorSegmentMB();
74+
}
75+
76+
public void setMaxMergeAtOnce(int maxMergeAtOnce) {
77+
regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
78+
forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
79+
}
80+
81+
public int getMaxMergeAtOnce() {
82+
return regularMergePolicy.getMaxMergeAtOnce();
83+
}
84+
85+
public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) {
86+
regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
87+
forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
88+
}
89+
90+
public int getMaxMergeAtOnceExplicit() {
91+
return forcedMergePolicy.getMaxMergeAtOnceExplicit();
92+
}
93+
94+
// only setter that must NOT delegate to the forced merge policy
95+
public void setMaxMergedSegmentMB(double mbFrac) {
96+
regularMergePolicy.setMaxMergedSegmentMB(mbFrac);
97+
}
98+
99+
public double getMaxMergedSegmentMB() {
100+
return regularMergePolicy.getMaxMergedSegmentMB();
101+
}
102+
103+
public void setSegmentsPerTier(double segmentsPerTier) {
104+
regularMergePolicy.setSegmentsPerTier(segmentsPerTier);
105+
forcedMergePolicy.setSegmentsPerTier(segmentsPerTier);
106+
}
107+
108+
public double getSegmentsPerTier() {
109+
return regularMergePolicy.getSegmentsPerTier();
110+
}
111+
112+
public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
113+
regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
114+
forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
115+
}
116+
117+
public double getReclaimDeletesWeight() {
118+
return regularMergePolicy.getReclaimDeletesWeight();
119+
}
120+
}

‎server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@
115115
*/
116116

117117
public final class MergePolicyConfig {
118-
private final TieredMergePolicy mergePolicy = new TieredMergePolicy();
118+
private final EsTieredMergePolicy mergePolicy = new EsTieredMergePolicy();
119119
private final Logger logger;
120120
private final boolean mergesEnabled;
121121

‎server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.index;
21+
22+
import org.apache.lucene.index.TieredMergePolicy;
23+
import org.elasticsearch.test.ESTestCase;
24+
25+
public class EsTieredMergePolicyTests extends ESTestCase {
26+
27+
public void testDefaults() {
28+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
29+
assertEquals(
30+
new TieredMergePolicy().getMaxMergedSegmentMB(),
31+
policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
32+
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
33+
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
34+
}
35+
36+
public void testSetMaxMergedSegmentMB() {
37+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
38+
policy.setMaxMergedSegmentMB(10 * 1024);
39+
assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
40+
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
41+
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
42+
}
43+
44+
public void testSetForceMergeDeletesPctAllowed() {
45+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
46+
policy.setForceMergeDeletesPctAllowed(42);
47+
assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0);
48+
}
49+
50+
public void testSetFloorSegmentMB() {
51+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
52+
policy.setFloorSegmentMB(42);
53+
assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0);
54+
assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0);
55+
}
56+
57+
public void testSetMaxMergeAtOnce() {
58+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
59+
policy.setMaxMergeAtOnce(42);
60+
assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce());
61+
}
62+
63+
public void testSetMaxMergeAtOnceExplicit() {
64+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
65+
policy.setMaxMergeAtOnceExplicit(42);
66+
assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit());
67+
}
68+
69+
public void testSetSegmentsPerTier() {
70+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
71+
policy.setSegmentsPerTier(42);
72+
assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
73+
}
74+
75+
public void testSetReclaimDeletesWeight() {
76+
EsTieredMergePolicy policy = new EsTieredMergePolicy();
77+
policy.setReclaimDeletesWeight(42);
78+
assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
79+
}
80+
}

‎server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
package org.elasticsearch.index;
2020

2121
import org.apache.lucene.index.NoMergePolicy;
22-
import org.apache.lucene.index.TieredMergePolicy;
2322
import org.elasticsearch.common.settings.Settings;
2423
import org.elasticsearch.common.unit.ByteSizeUnit;
2524
import org.elasticsearch.common.unit.ByteSizeValue;
@@ -76,43 +75,43 @@ public void testUpdateSettings() throws IOException {
7675

7776
public void testTieredMergePolicySettingsUpdate() throws IOException {
7877
IndexSettings indexSettings = indexSettings(Settings.EMPTY);
79-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
78+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
8079

8180
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build()));
82-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
81+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
8382

84-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
83+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
8584
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build()));
86-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
85+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
8786

88-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
87+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
8988
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build()));
90-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
89+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
9190

92-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
91+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
9392
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build()));
94-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
93+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
9594

96-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
95+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
9796
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
98-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
97+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
9998

100-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
99+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
101100
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
102-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
101+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
103102

104-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
103+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
105104
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
106-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
105+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
107106

108107
indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored
109-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
110-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
111-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
112-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
113-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
114-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
115-
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
108+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
109+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
110+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
111+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
112+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
113+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
114+
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
116115
}
117116

118117
public Settings build(String value) {

0 commit comments

Comments
 (0)