Skip to content

Commit ef2dc57

Browse files
HyunSangHanHyunSangHan
committed
Remove searchable_snapshot.extended_compatibility experimental feature
Signed-off-by: Hyunsang Han <gustkd3@gmail.com>
1 parent 57d1d17 commit ef2dc57

File tree

11 files changed

+8
-247
lines changed

11 files changed

+8
-247
lines changed

‎server/src/main/java/org/opensearch/common/util/FeatureFlags.java‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,6 @@ public class FeatureFlags {
3636
*/
3737
public static final String REMOTE_STORE_MIGRATION_EXPERIMENTAL = FEATURE_FLAG_PREFIX + "remote_store.migration.enabled";
3838

39-
/**
40-
* Gates the ability for Searchable Snapshots to read snapshots that are older than the
41-
* guaranteed backward compatibility for OpenSearch (one prior major version) on a best effort basis.
42-
*/
43-
public static final String SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY = FEATURE_FLAG_PREFIX
44-
+ "searchable_snapshot.extended_compatibility.enabled";
45-
public static final Setting<Boolean> SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_SETTING = Setting.boolSetting(
46-
SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY,
47-
false,
48-
Property.NodeScope
49-
);
50-
5139
/**
5240
* Gates the functionality of extensions.
5341
* Once the feature is ready for production release, this feature flag can be removed.
@@ -153,10 +141,6 @@ static class FeatureFlagsImpl {
153141
);
154142
put(TERM_VERSION_PRECOMMIT_ENABLE_SETTING, TERM_VERSION_PRECOMMIT_ENABLE_SETTING.getDefault(Settings.EMPTY));
155143
put(ARROW_STREAMS_SETTING, ARROW_STREAMS_SETTING.getDefault(Settings.EMPTY));
156-
put(
157-
SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_SETTING,
158-
SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_SETTING.getDefault(Settings.EMPTY)
159-
);
160144
put(MERGED_SEGMENT_WARMER_EXPERIMENTAL_SETTING, MERGED_SEGMENT_WARMER_EXPERIMENTAL_SETTING.getDefault(Settings.EMPTY));
161145
}
162146
};

‎server/src/main/java/org/opensearch/index/IndexSettings.java‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import org.opensearch.common.settings.Setting.Property;
4444
import org.opensearch.common.settings.Settings;
4545
import org.opensearch.common.unit.TimeValue;
46-
import org.opensearch.common.util.FeatureFlags;
4746
import org.opensearch.core.common.Strings;
4847
import org.opensearch.core.common.unit.ByteSizeUnit;
4948
import org.opensearch.core.common.unit.ByteSizeValue;
@@ -70,14 +69,12 @@
7069
import java.util.function.UnaryOperator;
7170

7271
import static org.opensearch.Version.V_2_7_0;
73-
import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
7472
import static org.opensearch.index.codec.fuzzy.FuzzySetParameters.DEFAULT_FALSE_POSITIVE_PROBABILITY;
7573
import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING;
7674
import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING;
7775
import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
7876
import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
7977
import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
80-
import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
8178
import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_DEFAULT_SLICE_COUNT_VALUE;
8279
import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MIN_SLICE_COUNT_VALUE;
8380
import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL;
@@ -805,7 +802,6 @@ public static IndexMergePolicy fromString(String text) {
805802
private volatile String remoteStoreRepository;
806803
private int remoteTranslogKeepExtraGen;
807804
private boolean autoForcemergeEnabled;
808-
private Version extendedCompatibilitySnapshotVersion;
809805

810806
// volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
811807
private volatile Settings settings;
@@ -1019,13 +1015,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
10191015
remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
10201016
remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY);
10211017
this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings);
1022-
1023-
if (isRemoteSnapshot() && FeatureFlags.isEnabled(SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY)) {
1024-
extendedCompatibilitySnapshotVersion = SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
1025-
} else {
1026-
extendedCompatibilitySnapshotVersion = Version.CURRENT.minimumIndexCompatibilityVersion();
1027-
}
1028-
10291018
this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
10301019
this.shouldCleanupUnreferencedFiles = INDEX_UNREFERENCED_FILE_CLEANUP.get(settings);
10311020
this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -1438,16 +1427,6 @@ public boolean isRemoteSnapshot() {
14381427
return indexMetadata.isRemoteSnapshot();
14391428
}
14401429

1441-
/**
1442-
* If this is a remote snapshot and the extended compatibility
1443-
* feature flag is enabled, this returns the minimum {@link Version}
1444-
* supported. In all other cases, the return value is the
1445-
* {@link Version#minimumIndexCompatibilityVersion()} of {@link Version#CURRENT}.
1446-
*/
1447-
public Version getExtendedCompatibilitySnapshotVersion() {
1448-
return extendedCompatibilitySnapshotVersion;
1449-
}
1450-
14511430
/**
14521431
* Returns the node settings. The settings returned from {@link #getSettings()} are a merged version of the
14531432
* index settings and the node settings where node settings are overwritten by index settings.

‎server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import org.apache.lucene.search.ReferenceManager;
4040
import org.apache.lucene.store.Directory;
4141
import org.apache.lucene.store.Lock;
42-
import org.opensearch.Version;
4342
import org.opensearch.cluster.metadata.IndexMetadata;
4443
import org.opensearch.common.concurrent.GatedCloseable;
4544
import org.opensearch.common.lucene.Lucene;
@@ -93,7 +92,6 @@ public class ReadOnlyEngine extends Engine {
9392
private final CompletionStatsCache completionStatsCache;
9493
private final boolean requireCompleteHistory;
9594
private final TranslogManager translogManager;
96-
private final Version minimumSupportedVersion;
9795

9896
protected volatile TranslogStats translogStats;
9997

@@ -120,8 +118,6 @@ public ReadOnlyEngine(
120118
) {
121119
super(config);
122120
this.requireCompleteHistory = requireCompleteHistory;
123-
// fetch the minimum Version for extended backward compatibility use-cases
124-
this.minimumSupportedVersion = config.getIndexSettings().getExtendedCompatibilitySnapshotVersion();
125121
try {
126122
Store store = config.getStore();
127123
store.incRef();
@@ -133,11 +129,7 @@ public ReadOnlyEngine(
133129
// we obtain the IW lock even though we never modify the index.
134130
// yet this makes sure nobody else does. including some testing tools that try to be messy
135131
indexWriterLock = obtainLock ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : null;
136-
if (isExtendedCompatibility()) {
137-
this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory, this.minimumSupportedVersion);
138-
} else {
139-
this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
140-
}
132+
this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
141133
if (seqNoStats == null) {
142134
seqNoStats = buildSeqNoStats(config, lastCommittedSegmentInfos);
143135
ensureMaxSeqNoEqualsToGlobalCheckpoint(seqNoStats);
@@ -221,19 +213,10 @@ protected final OpenSearchDirectoryReader wrapReader(
221213

222214
protected DirectoryReader open(IndexCommit commit) throws IOException {
223215
assert Transports.assertNotTransportThread("opening index commit of a read-only engine");
224-
DirectoryReader reader;
225-
if (isExtendedCompatibility()) {
226-
reader = DirectoryReader.open(commit, this.minimumSupportedVersion.luceneVersion.major, null);
227-
} else {
228-
reader = DirectoryReader.open(commit);
229-
}
216+
DirectoryReader reader = DirectoryReader.open(commit);
230217
return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
231218
}
232219

233-
private boolean isExtendedCompatibility() {
234-
return Version.CURRENT.minimumIndexCompatibilityVersion().onOrAfter(this.minimumSupportedVersion);
235-
}
236-
237220
@Override
238221
protected void closeNoLock(String reason, CountDownLatch closedLatch) {
239222
if (isClosed.compareAndSet(false, true)) {

‎server/src/main/java/org/opensearch/index/shard/IndexShard.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2772,11 +2772,7 @@ private boolean assertSequenceNumbersInCommit() throws IOException {
27722772
}
27732773

27742774
private Map<String, String> fetchUserData() throws IOException {
2775-
if (indexSettings.isRemoteSnapshot() && indexSettings.getExtendedCompatibilitySnapshotVersion() != null) {
2776-
return Lucene.readSegmentInfos(store.directory(), indexSettings.getExtendedCompatibilitySnapshotVersion()).getUserData();
2777-
} else {
2778-
return SegmentInfos.readLatestCommit(store.directory()).getUserData();
2779-
}
2775+
return SegmentInfos.readLatestCommit(store.directory()).getUserData();
27802776
}
27812777

27822778
private void onNewEngine(Engine newEngine) {

‎server/src/main/java/org/opensearch/index/store/Store.java‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -231,11 +231,7 @@ public ShardPath shardPath() {
231231
public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
232232
failIfCorrupted();
233233
try {
234-
if (indexSettings.isRemoteSnapshot() && indexSettings.getExtendedCompatibilitySnapshotVersion() != null) {
235-
return readSegmentInfosExtendedCompatibility(directory(), indexSettings.getExtendedCompatibilitySnapshotVersion());
236-
} else {
237-
return readSegmentsInfo(null, directory());
238-
}
234+
return readSegmentsInfo(null, directory());
239235
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
240236
markStoreCorrupted(ex);
241237
throw ex;
@@ -245,7 +241,7 @@ public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
245241
/**
246242
* Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code>.
247243
* This method will throw an exception if the index is older than the standard backwards compatibility
248-
* policy ( current major - 1). See also {@link #readSegmentInfosExtendedCompatibility(Directory, org.opensearch.Version)}.
244+
* policy ( current major - 1).
249245
*
250246
* @throws IOException if the index is corrupted or the segments file is not present
251247
*/
@@ -263,27 +259,6 @@ private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory direc
263259
}
264260
}
265261

266-
/**
267-
* Returns the segments info for the latest commit in the given directory. Unlike
268-
* {@link #readSegmentsInfo(IndexCommit, Directory)}, this method supports reading
269-
* older Lucene indices on a best-effort basis.
270-
*
271-
* @throws IOException if the index is corrupted or the segments file is not present
272-
*/
273-
private static SegmentInfos readSegmentInfosExtendedCompatibility(Directory directory, org.opensearch.Version minimumVersion)
274-
throws IOException {
275-
try {
276-
return Lucene.readSegmentInfos(directory, minimumVersion);
277-
} catch (EOFException eof) {
278-
// TODO this should be caught by lucene - EOF is almost certainly an index corruption
279-
throw new CorruptIndexException("Read past EOF while reading segment infos", "<latest-commit>", eof);
280-
} catch (IOException exception) {
281-
throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up
282-
} catch (Exception ex) {
283-
throw new CorruptIndexException("Hit unexpected exception while reading segment infos", "<latest-commit>", ex);
284-
}
285-
}
286-
287262
final void ensureOpen() {
288263
if (this.refCounter.refCount() <= 0) {
289264
throw new AlreadyClosedException("store is already closed");

‎server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectory.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515
import org.apache.lucene.store.IndexOutput;
1616
import org.apache.lucene.store.Lock;
1717
import org.apache.lucene.store.NoLockFactory;
18-
import org.opensearch.LegacyESVersion;
19-
import org.opensearch.Version;
2018
import org.opensearch.common.lucene.store.ByteArrayIndexInput;
2119
import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
2220
import org.opensearch.index.store.remote.file.OnDemandBlockSnapshotIndexInput;
@@ -38,8 +36,6 @@
3836
*/
3937
public final class RemoteSnapshotDirectory extends Directory {
4038

41-
public static final Version SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION = LegacyESVersion.V_7_2_0;
42-
4339
private static final String VIRTUAL_FILE_PREFIX = BlobStoreRepository.VIRTUAL_DATA_BLOB_PREFIX;
4440

4541
private final Map<String, BlobStoreIndexShardSnapshot.FileInfo> fileInfoMap;

‎server/src/main/java/org/opensearch/snapshots/RestoreService.java‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@
8080
import org.opensearch.common.settings.Settings;
8181
import org.opensearch.common.unit.TimeValue;
8282
import org.opensearch.common.util.ArrayUtils;
83-
import org.opensearch.common.util.FeatureFlags;
8483
import org.opensearch.core.action.ActionListener;
8584
import org.opensearch.core.index.Index;
8685
import org.opensearch.core.index.shard.ShardId;
@@ -126,11 +125,9 @@
126125
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
127126
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
128127
import static org.opensearch.cluster.service.ClusterManagerTask.RESTORE_SNAPSHOT;
129-
import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
130128
import static org.opensearch.common.util.IndexUtils.filterIndices;
131129
import static org.opensearch.common.util.set.Sets.newHashSet;
132130
import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
133-
import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
134131
import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING;
135132

136133
/**
@@ -434,15 +431,9 @@ public ClusterState execute(ClusterState currentState) {
434431
request.getSourceRemoteTranslogRepository(),
435432
snapshotInfo.getPinnedTimestamp()
436433
);
437-
final Version minIndexCompatibilityVersion;
438-
if (isSearchableSnapshot && isSearchableSnapshotsExtendedCompatibilityEnabled()) {
439-
minIndexCompatibilityVersion = SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION
440-
.minimumIndexCompatibilityVersion();
441-
} else {
442-
minIndexCompatibilityVersion = currentState.getNodes()
443-
.getMaxNodeVersion()
444-
.minimumIndexCompatibilityVersion();
445-
}
434+
final Version minIndexCompatibilityVersion = currentState.getNodes()
435+
.getMaxNodeVersion()
436+
.minimumIndexCompatibilityVersion();
446437
try {
447438
snapshotIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(
448439
snapshotIndexMetadata,
@@ -1390,9 +1381,4 @@ private static IndexMetadata addSnapshotToIndexSettings(IndexMetadata metadata,
13901381
.build();
13911382
return IndexMetadata.builder(metadata).settings(newSettings).build();
13921383
}
1393-
1394-
private static boolean isSearchableSnapshotsExtendedCompatibilityEnabled() {
1395-
return org.opensearch.Version.CURRENT.after(org.opensearch.Version.V_2_4_0)
1396-
&& FeatureFlags.isEnabled(SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY);
1397-
}
13981384
}

‎server/src/test/java/org/opensearch/common/lucene/LuceneTests.java‎

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,10 @@
3636
import org.apache.lucene.document.Field;
3737
import org.apache.lucene.document.Field.Store;
3838
import org.apache.lucene.document.LatLonDocValuesField;
39-
import org.apache.lucene.document.LatLonPoint;
4039
import org.apache.lucene.document.NumericDocValuesField;
4140
import org.apache.lucene.document.StringField;
4241
import org.apache.lucene.document.TextField;
4342
import org.apache.lucene.index.DirectoryReader;
44-
import org.apache.lucene.index.IndexCommit;
45-
import org.apache.lucene.index.IndexFormatTooOldException;
4643
import org.apache.lucene.index.IndexReader;
4744
import org.apache.lucene.index.IndexWriter;
4845
import org.apache.lucene.index.IndexWriterConfig;
@@ -51,7 +48,6 @@
5148
import org.apache.lucene.index.NoMergePolicy;
5249
import org.apache.lucene.index.SegmentInfos;
5350
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
54-
import org.apache.lucene.index.StandardDirectoryReader;
5551
import org.apache.lucene.index.StoredFields;
5652
import org.apache.lucene.index.Term;
5753
import org.apache.lucene.search.Explanation;
@@ -75,11 +71,8 @@
7571
import org.apache.lucene.tests.analysis.MockAnalyzer;
7672
import org.apache.lucene.tests.index.RandomIndexWriter;
7773
import org.apache.lucene.tests.store.MockDirectoryWrapper;
78-
import org.apache.lucene.tests.util.TestUtil;
7974
import org.apache.lucene.util.Bits;
8075
import org.apache.lucene.util.BytesRef;
81-
import org.opensearch.LegacyESVersion;
82-
import org.opensearch.Version;
8376
import org.opensearch.common.collect.Tuple;
8477
import org.opensearch.common.util.io.IOUtils;
8578
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -95,7 +88,6 @@
9588

9689
import java.io.IOException;
9790
import java.io.StringReader;
98-
import java.nio.file.Path;
9991
import java.util.ArrayList;
10092
import java.util.Collections;
10193
import java.util.HashSet;
@@ -329,36 +321,6 @@ public void testNumDocs() throws IOException {
329321
dir.close();
330322
}
331323

332-
/**
333-
* Tests whether old segments are readable and queryable based on the data documented
334-
* in the README <a href="file:../../../../../resources/indices/bwc/os-1.3.0/README.md">here</a>.
335-
*/
336-
public void testReadSegmentInfosExtendedCompatibility() throws IOException {
337-
final Version minVersion = LegacyESVersion.V_7_2_0;
338-
Path tmp = createTempDir();
339-
TestUtil.unzip(getClass().getResourceAsStream(OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
340-
try (MockDirectoryWrapper dir = newMockFSDirectory(tmp)) {
341-
// The standard API will throw an exception
342-
expectThrows(IndexFormatTooOldException.class, () -> Lucene.readSegmentInfos(dir));
343-
SegmentInfos si = Lucene.readSegmentInfos(dir, minVersion);
344-
assertEquals(1, Lucene.getNumDocs(si));
345-
IndexCommit indexCommit = Lucene.getIndexCommit(si, dir);
346-
// uses the "expert" Lucene API
347-
try (
348-
StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(
349-
indexCommit,
350-
minVersion.minimumIndexCompatibilityVersion().luceneVersion.major,
351-
null
352-
)
353-
) {
354-
IndexSearcher searcher = newSearcher(reader);
355-
// radius too small, should get no results
356-
assertFalse(Lucene.exists(searcher, LatLonPoint.newDistanceQuery("testLocation", 48.57532, -112.87695, 2)));
357-
assertTrue(Lucene.exists(searcher, LatLonPoint.newDistanceQuery("testLocation", 48.57532, -112.87695, 20000)));
358-
}
359-
}
360-
}
361-
362324
public void testCount() throws Exception {
363325
Directory dir = newDirectory();
364326
RandomIndexWriter w = new RandomIndexWriter(random(), dir);

0 commit comments

Comments
 (0)