From 5f7d3f9c34daa74f88568b22c12a5eaf0ef1119b Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Thu, 13 Jan 2022 23:56:57 -0600 Subject: [PATCH] remove straggler soft delete disabled tests Signed-off-by: Nicholas Walter Knize --- .../test/indices.create/10_basic.yml | 13 - .../index/engine/InternalEngineTests.java | 249 ------------------ 2 files changed, 262 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index e61b273cdb832..30419f7738bbf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -155,16 +155,3 @@ - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." } ---- -"Create index without soft deletes": - - skip: - version: " - 7.9.99" - reason: "indices without soft deletes is no longer supported" - - - do: - catch: /illegal_argument_exception/ - indices.create: - index: test_index - body: - settings: - soft_deletes.enabled: false diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index a9eeedde31f49..0a7dda8c60341 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -313,202 +313,6 @@ public void testVersionMapAfterAutoIDDocument() throws IOException { } } - public void testSegmentsWithoutSoftDeletes() throws Exception { - Settings settings = Settings.builder() - .put(defaultSettings.getSettings()) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .build(); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( - IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build() - ); - try ( - Store store = createStore(); - InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null)) - ) { - List segments = engine.segments(false); - assertThat(segments.isEmpty(), equalTo(true)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(0L)); - assertThat(engine.segmentsStats(false, false).getMemoryInBytes(), equalTo(0L)); - - // create two docs and refresh - ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null); - Engine.Index first = indexForDoc(doc); - Engine.IndexResult firstResult = engine.index(first); - ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null); - Engine.Index second = indexForDoc(doc2); - Engine.IndexResult secondResult = engine.index(second); - assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation())); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(1)); - SegmentsStats stats = engine.segmentsStats(false, false); - assertThat(stats.getCount(), equalTo(1L)); - assertThat(stats.getTermsMemoryInBytes(), greaterThan(0L)); - assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L)); - assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L)); - assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L)); - assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L)); - assertThat(segments.get(0).isCommitted(), equalTo(false)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(2)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - assertThat(segments.get(0).ramTree, nullValue()); - engine.flush(); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(1)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(1L)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(2)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - ParsedDocument doc3 = testParsedDocument("3", null, testDocumentWithTextField(), B_3, null); - engine.index(indexForDoc(doc3)); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(2)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L)); - assertThat(engine.segmentsStats(false, false).getTermsMemoryInBytes(), greaterThan(stats.getTermsMemoryInBytes())); - assertThat( - engine.segmentsStats(false, false).getStoredFieldsMemoryInBytes(), - greaterThan(stats.getStoredFieldsMemoryInBytes()) - ); - assertThat(engine.segmentsStats(false, false).getTermVectorsMemoryInBytes(), equalTo(0L)); - assertThat(engine.segmentsStats(false, false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); - assertThat(engine.segmentsStats(false, false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(2)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get())); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(2)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - engine.onSettingsChanged( - indexSettings.getTranslogRetentionAge(), - indexSettings.getTranslogRetentionSize(), - indexSettings.getSoftDeleteRetentionOperations() - ); - ParsedDocument doc4 = testParsedDocument("4", null, testDocumentWithTextField(), B_3, null); - engine.index(indexForDoc(doc4)); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(3)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(3L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - assertThat(segments.get(2).isCommitted(), equalTo(false)); - assertThat(segments.get(2).isSearch(), equalTo(true)); - assertThat(segments.get(2).getNumDocs(), equalTo(1)); - assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(2).isCompound(), equalTo(true)); - - // internal refresh - lets make sure we see those segments in the stats - ParsedDocument doc5 = testParsedDocument("5", null, testDocumentWithTextField(), B_3, null); - engine.index(indexForDoc(doc5)); - engine.refresh("test", Engine.SearcherScope.INTERNAL, true); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(4)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - assertThat(segments.get(2).isCommitted(), equalTo(false)); - assertThat(segments.get(2).isSearch(), equalTo(true)); - assertThat(segments.get(2).getNumDocs(), equalTo(1)); - assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(2).isCompound(), equalTo(true)); - - assertThat(segments.get(3).isCommitted(), equalTo(false)); - assertThat(segments.get(3).isSearch(), equalTo(false)); - assertThat(segments.get(3).getNumDocs(), equalTo(1)); - assertThat(segments.get(3).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(3).isCompound(), equalTo(true)); - - // now refresh the external searcher and make sure it has the new segment - engine.refresh("test"); - segments = engine.segments(false); - assertThat(segments.size(), equalTo(4)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - assertThat(segments.get(2).isCommitted(), equalTo(false)); - assertThat(segments.get(2).isSearch(), equalTo(true)); - assertThat(segments.get(2).getNumDocs(), equalTo(1)); - assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(2).isCompound(), equalTo(true)); - - assertThat(segments.get(3).isCommitted(), equalTo(false)); - assertThat(segments.get(3).isSearch(), equalTo(true)); - assertThat(segments.get(3).getNumDocs(), equalTo(1)); - assertThat(segments.get(3).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(3).isCompound(), equalTo(true)); - } - } - public void testVerboseSegments() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { List segments = engine.segments(true); @@ -1688,59 +1492,6 @@ public void testVersioningNewIndex() throws IOException { assertThat(indexResult.getVersion(), equalTo(1L)); } - public void testForceMergeWithoutSoftDeletes() throws IOException { - Settings settings = Settings.builder() - .put(defaultSettings.getSettings()) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .build(); - IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build(); - try ( - Store store = createStore(); - Engine engine = createEngine( - config(IndexSettingsModule.newIndexSettings(indexMetadata), store, createTempDir(), new LogByteSizeMergePolicy(), null) - ) - ) { // use log MP here we test some behavior in ESMP - int numDocs = randomIntBetween(10, 100); - for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null); - Engine.Index index = indexForDoc(doc); - engine.index(index); - engine.refresh("test"); - } - try (Engine.Searcher test = engine.acquireSearcher("test")) { - assertEquals(numDocs, test.getIndexReader().numDocs()); - } - engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID()); - engine.refresh("test"); - assertEquals(engine.segments(true).size(), 1); - - ParsedDocument doc = testParsedDocument(Integer.toString(0), null, testDocument(), B_1, null); - Engine.Index index = indexForDoc(doc); - engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get())); - // expunge deletes - engine.forceMerge(true, 10, true, false, false, UUIDs.randomBase64UUID()); - engine.refresh("test"); - - assertEquals(engine.segments(true).size(), 1); - try (Engine.Searcher test = engine.acquireSearcher("test")) { - assertEquals(numDocs - 1, test.getIndexReader().numDocs()); - assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.getIndexReader().maxDoc()); - } - - doc = testParsedDocument(Integer.toString(1), null, testDocument(), B_1, null); - index = indexForDoc(doc); - engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get())); - // expunge deletes - engine.forceMerge(true, 10, false, false, false, UUIDs.randomBase64UUID()); - engine.refresh("test"); - assertEquals(engine.segments(true).size(), 1); - try (Engine.Searcher test = engine.acquireSearcher("test")) { - assertEquals(numDocs - 2, test.getIndexReader().numDocs()); - assertEquals(numDocs - 1, test.getIndexReader().maxDoc()); - } - } - } - /* * we are testing an edge case here where we have a fully deleted segment that is retained but has all it's IDs pruned away. */