Skip to content

Commit

Permalink
remove straggler soft delete disabled tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
  • Loading branch information
@nknize
nknize committed Jan 14, 2022
1 parent 7f42949 commit 5f7d3f9
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 262 deletions.
13 changes: 0 additions & 13 deletions rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,16 +155,3 @@
- match: { error.type: "illegal_argument_exception" }
- match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }

---
"Create index without soft deletes":
- skip:
version: " - 7.9.99"
reason: "indices without soft deletes is no longer supported"

- do:
catch: /illegal_argument_exception/
indices.create:
index: test_index
body:
settings:
soft_deletes.enabled: false
249 changes: 0 additions & 249 deletions server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,202 +313,6 @@ public void testVersionMapAfterAutoIDDocument() throws IOException {
}
}

public void testSegmentsWithoutSoftDeletes() throws Exception {
Settings settings = Settings.builder()
.put(defaultSettings.getSettings())
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
.build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build()
);
try (
Store store = createStore();
InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))
) {
List<Segment> segments = engine.segments(false);
assertThat(segments.isEmpty(), equalTo(true));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(0L));
assertThat(engine.segmentsStats(false, false).getMemoryInBytes(), equalTo(0L));

// create two docs and refresh
ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
Engine.Index first = indexForDoc(doc);
Engine.IndexResult firstResult = engine.index(first);
ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null);
Engine.Index second = indexForDoc(doc2);
Engine.IndexResult secondResult = engine.index(second);
assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation()));
engine.refresh("test");

segments = engine.segments(false);
assertThat(segments.size(), equalTo(1));
SegmentsStats stats = engine.segmentsStats(false, false);
assertThat(stats.getCount(), equalTo(1L));
assertThat(stats.getTermsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L));
assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L));
assertThat(segments.get(0).isCommitted(), equalTo(false));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(2));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertThat(segments.get(0).isCompound(), equalTo(true));
assertThat(segments.get(0).ramTree, nullValue());
engine.flush();

segments = engine.segments(false);
assertThat(segments.size(), equalTo(1));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(1L));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(2));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertThat(segments.get(0).isCompound(), equalTo(true));

ParsedDocument doc3 = testParsedDocument("3", null, testDocumentWithTextField(), B_3, null);
engine.index(indexForDoc(doc3));
engine.refresh("test");

segments = engine.segments(false);
assertThat(segments.size(), equalTo(2));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L));
assertThat(engine.segmentsStats(false, false).getTermsMemoryInBytes(), greaterThan(stats.getTermsMemoryInBytes()));
assertThat(
engine.segmentsStats(false, false).getStoredFieldsMemoryInBytes(),
greaterThan(stats.getStoredFieldsMemoryInBytes())
);
assertThat(engine.segmentsStats(false, false).getTermVectorsMemoryInBytes(), equalTo(0L));
assertThat(engine.segmentsStats(false, false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
assertThat(engine.segmentsStats(false, false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(2));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertThat(segments.get(0).isCompound(), equalTo(true));

assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(true));

engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get()));
engine.refresh("test");

segments = engine.segments(false);
assertThat(segments.size(), equalTo(2));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(1));
assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
assertThat(segments.get(0).isCompound(), equalTo(true));

assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(true));

engine.onSettingsChanged(
indexSettings.getTranslogRetentionAge(),
indexSettings.getTranslogRetentionSize(),
indexSettings.getSoftDeleteRetentionOperations()
);
ParsedDocument doc4 = testParsedDocument("4", null, testDocumentWithTextField(), B_3, null);
engine.index(indexForDoc(doc4));
engine.refresh("test");

segments = engine.segments(false);
assertThat(segments.size(), equalTo(3));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(3L));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(1));
assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
assertThat(segments.get(0).isCompound(), equalTo(true));

assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(true));

assertThat(segments.get(2).isCommitted(), equalTo(false));
assertThat(segments.get(2).isSearch(), equalTo(true));
assertThat(segments.get(2).getNumDocs(), equalTo(1));
assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
assertThat(segments.get(2).isCompound(), equalTo(true));

// internal refresh - lets make sure we see those segments in the stats
ParsedDocument doc5 = testParsedDocument("5", null, testDocumentWithTextField(), B_3, null);
engine.index(indexForDoc(doc5));
engine.refresh("test", Engine.SearcherScope.INTERNAL, true);

segments = engine.segments(false);
assertThat(segments.size(), equalTo(4));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(1));
assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
assertThat(segments.get(0).isCompound(), equalTo(true));

assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(true));

assertThat(segments.get(2).isCommitted(), equalTo(false));
assertThat(segments.get(2).isSearch(), equalTo(true));
assertThat(segments.get(2).getNumDocs(), equalTo(1));
assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
assertThat(segments.get(2).isCompound(), equalTo(true));

assertThat(segments.get(3).isCommitted(), equalTo(false));
assertThat(segments.get(3).isSearch(), equalTo(false));
assertThat(segments.get(3).getNumDocs(), equalTo(1));
assertThat(segments.get(3).getDeletedDocs(), equalTo(0));
assertThat(segments.get(3).isCompound(), equalTo(true));

// now refresh the external searcher and make sure it has the new segment
engine.refresh("test");
segments = engine.segments(false);
assertThat(segments.size(), equalTo(4));
assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(1));
assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
assertThat(segments.get(0).isCompound(), equalTo(true));

assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(true));

assertThat(segments.get(2).isCommitted(), equalTo(false));
assertThat(segments.get(2).isSearch(), equalTo(true));
assertThat(segments.get(2).getNumDocs(), equalTo(1));
assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
assertThat(segments.get(2).isCompound(), equalTo(true));

assertThat(segments.get(3).isCommitted(), equalTo(false));
assertThat(segments.get(3).isSearch(), equalTo(true));
assertThat(segments.get(3).getNumDocs(), equalTo(1));
assertThat(segments.get(3).getDeletedDocs(), equalTo(0));
assertThat(segments.get(3).isCompound(), equalTo(true));
}
}

public void testVerboseSegments() throws Exception {
try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
List<Segment> segments = engine.segments(true);
Expand Down Expand Up @@ -1688,59 +1492,6 @@ public void testVersioningNewIndex() throws IOException {
assertThat(indexResult.getVersion(), equalTo(1L));
}

public void testForceMergeWithoutSoftDeletes() throws IOException {
Settings settings = Settings.builder()
.put(defaultSettings.getSettings())
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
.build();
IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
try (
Store store = createStore();
Engine engine = createEngine(
config(IndexSettingsModule.newIndexSettings(indexMetadata), store, createTempDir(), new LogByteSizeMergePolicy(), null)
)
) { // use log MP here we test some behavior in ESMP
int numDocs = randomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null);
Engine.Index index = indexForDoc(doc);
engine.index(index);
engine.refresh("test");
}
try (Engine.Searcher test = engine.acquireSearcher("test")) {
assertEquals(numDocs, test.getIndexReader().numDocs());
}
engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID());
engine.refresh("test");
assertEquals(engine.segments(true).size(), 1);

ParsedDocument doc = testParsedDocument(Integer.toString(0), null, testDocument(), B_1, null);
Engine.Index index = indexForDoc(doc);
engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get()));
// expunge deletes
engine.forceMerge(true, 10, true, false, false, UUIDs.randomBase64UUID());
engine.refresh("test");

assertEquals(engine.segments(true).size(), 1);
try (Engine.Searcher test = engine.acquireSearcher("test")) {
assertEquals(numDocs - 1, test.getIndexReader().numDocs());
assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.getIndexReader().maxDoc());
}

doc = testParsedDocument(Integer.toString(1), null, testDocument(), B_1, null);
index = indexForDoc(doc);
engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get()));
// expunge deletes
engine.forceMerge(true, 10, false, false, false, UUIDs.randomBase64UUID());
engine.refresh("test");
assertEquals(engine.segments(true).size(), 1);
try (Engine.Searcher test = engine.acquireSearcher("test")) {
assertEquals(numDocs - 2, test.getIndexReader().numDocs());
assertEquals(numDocs - 1, test.getIndexReader().maxDoc());
}
}
}

/*
* we are testing an edge case here where we have a fully deleted segment that is retained but has all it's IDs pruned away.
*/
Expand Down

0 comments on commit 5f7d3f9

Please sign in to comment.