diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 32a4d5d228726..fe9927653b19a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -36,7 +36,7 @@ * {@code _tsid} and {@code @timestamp}. */ public class TsidExtractingIdFieldMapper extends IdFieldMapper { - private static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE = new FieldType(); /** * Maximum length of the {@code _tsid} in the {@link #documentDescription}. */ diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index bdf1af996544d..3db1f2e1aa2ca 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -49,8 +49,8 @@ import org.elasticsearch.index.cache.query.IndexQueryCache; import org.elasticsearch.index.cache.query.QueryCache; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.engine.InternalEngineFactory; -import org.elasticsearch.index.engine.InternalEngineTests; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.ParsedDocument; @@ -306,7 +306,7 @@ public Engine.Index preIndex(ShardId shardId, Engine.Index operation) { assertEquals(IndexingSlowLog.class, indexService.getIndexOperationListeners().get(0).getClass()); assertSame(listener, indexService.getIndexOperationListeners().get(1)); - ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); + ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null); Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong(), doc); ShardId shardId = new ShardId(new Index("foo", "bar"), 0); for (IndexingOperationListener l : indexService.getIndexOperationListeners()) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index 6edf6f6647a1f..74e56e2c509ea 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexingSlowLog.IndexingSlowLogMessage; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.InternalEngineTests; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.Uid; @@ -74,7 +74,7 @@ public void testLevelPrecedence() { IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); IndexingSlowLog log = new IndexingSlowLog(settings); - ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); + ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null); Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId("doc_id")), randomNonNegativeLong(), doc); Engine.IndexResult result = Mockito.mock(Engine.IndexResult.class);// (0, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, false); Mockito.when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); @@ -148,7 +148,7 @@ public void testTwoLoggersDifferentLevel() { ); IndexingSlowLog log2 = new IndexingSlowLog(index2Settings); - ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); + ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null); Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId("doc_id")), randomNonNegativeLong(), doc); Engine.IndexResult result = Mockito.mock(Engine.IndexResult.class); Mockito.when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS); diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 22ddf52f32396..cae5a65ffb3cb 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -17,6 +17,7 @@ import org.apache.logging.log4j.core.appender.AbstractAppender; import org.apache.logging.log4j.core.filter.RegexFilter; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -107,6 +108,7 @@ import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.seqno.LocalCheckpointTracker; @@ -985,7 +987,7 @@ public void testSimpleOperations() throws Exception { public void testGetWithSearcherWrapper() throws Exception { engine.refresh("warm_up"); - engine.index(indexForDoc(createParsedDoc("1", null))); + engine.index(indexForDoc(createParsedDoc("1", idFieldType, null))); assertThat(engine.lastRefreshedCheckpoint(), equalTo(NO_OPS_PERFORMED)); MapperService mapperService = createMapperService(); MappingLookup mappingLookup = mapperService.mappingLookup(); @@ -1003,7 +1005,7 @@ public void testGetWithSearcherWrapper() throws Exception { // refresh triggered, as we did not track translog location until the first realtime get. assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L)); - engine.index(indexForDoc(createParsedDoc("1", null))); + engine.index(indexForDoc(createParsedDoc("1", idFieldType, null))); try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), mappingLookup, documentParser, searcher -> searcher)) { assertTrue(get.exists()); assertEquals(++translogGetCountExpected, translogGetCount.getAsLong()); @@ -1012,7 +1014,7 @@ public void testGetWithSearcherWrapper() throws Exception { assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L)); // no refresh; just read from translog if (randomBoolean()) { - engine.index(indexForDoc(createParsedDoc("1", null))); + engine.index(indexForDoc(createParsedDoc("1", idFieldType, null))); } try ( Engine.GetResult get = engine.get( @@ -1069,7 +1071,7 @@ public void testGetWithSearcherWrapper() throws Exception { } assertThat("no refresh, just read from translog or in-memory segment", engine.lastRefreshedCheckpoint(), equalTo(0L)); - engine.index(indexForDoc(createParsedDoc("1", null))); + engine.index(indexForDoc(createParsedDoc("1", idFieldType, null))); try ( Engine.GetResult get = engine.get( new Engine.Get(true, true, "1"), @@ -1517,6 +1519,7 @@ public void testVersioningNewIndex() throws IOException { * we are testing an edge case here where we have a fully deleted segment that is retained but has all it's IDs pruned away. */ public void testLookupVersionWithPrunedAwayIds() throws IOException { + FieldType idFieldType = randomBoolean() ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : TsidExtractingIdFieldMapper.FIELD_TYPE; try (Directory dir = newDirectory()) { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Lucene.STANDARD_ANALYZER); indexWriterConfig.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD); @@ -1533,7 +1536,7 @@ public void testLookupVersionWithPrunedAwayIds() throws IOException { ) ) { org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document(); - doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE)); + doc.add(new Field(IdFieldMapper.NAME, "1", idFieldType)); doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, -1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 1)); doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1)); @@ -1668,7 +1671,15 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc int numDocs = scaledRandomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { boolean useRecoverySource = randomBoolean() || omitSourceAllTheTime; - ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null, useRecoverySource); + ParsedDocument doc = testParsedDocument( + Integer.toString(i), + idFieldType, + null, + testDocument(), + B_1, + null, + useRecoverySource + ); engine.index(indexForDoc(doc)); liveDocs.add(doc.id()); if (useRecoverySource == false) { @@ -1677,7 +1688,15 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc } for (int i = 0; i < numDocs; i++) { boolean useRecoverySource = randomBoolean() || omitSourceAllTheTime; - ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null, useRecoverySource); + ParsedDocument doc = testParsedDocument( + Integer.toString(i), + idFieldType, + null, + testDocument(), + B_1, + null, + useRecoverySource + ); if (randomBoolean()) { engine.delete(new Engine.Delete(doc.id(), newUid(doc.id()), primaryTerm.get())); liveDocs.remove(doc.id()); @@ -1737,7 +1756,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc } if (numSegments == 1) { boolean useRecoverySource = randomBoolean() || omitSourceAllTheTime; - ParsedDocument doc = testParsedDocument("dummy", null, testDocument(), B_1, null, useRecoverySource); + ParsedDocument doc = testParsedDocument("dummy", idFieldType, null, testDocument(), B_1, null, useRecoverySource); engine.index(indexForDoc(doc)); if (useRecoverySource == false) { liveDocsWithSource.add(doc.id()); @@ -4626,7 +4645,7 @@ public void testLookupSeqNoByIdInLucene() throws Exception { boolean isIndexing = randomBoolean(); int copies = frequently() ? 1 : between(2, 4); for (int c = 0; c < copies; c++) { - final ParsedDocument doc = EngineTestCase.createParsedDoc(id, null); + final ParsedDocument doc = EngineTestCase.createParsedDoc(id, idFieldType, null); if (isIndexing) { operations.add( new Engine.Index( @@ -5067,7 +5086,7 @@ public void testRandomOperations() throws Exception { int numOps = between(10, 100); for (int i = 0; i < numOps; i++) { String id = Integer.toString(randomIntBetween(1, 10)); - ParsedDocument doc = createParsedDoc(id, null); + ParsedDocument doc = createParsedDoc(id, idFieldType, null); Engine.Operation.TYPE type = randomFrom(Engine.Operation.TYPE.values()); switch (type) { case INDEX -> { @@ -5467,7 +5486,11 @@ public void testSeqNoGenerator() throws IOException { ) ) { final String id = "id"; - final Field uidField = new Field("_id", id, ProvidedIdFieldMapper.Defaults.FIELD_TYPE); + final Field uidField = new Field( + "_id", + id, + randomBoolean() ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : TsidExtractingIdFieldMapper.FIELD_TYPE + ); final Field versionField = new NumericDocValuesField("_version", 0); final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); final LuceneDocument document = new LuceneDocument(); @@ -6454,7 +6477,7 @@ public void testTrackMaxSeqNoOfUpdatesOrDeletesOnPrimary() throws Exception { int numOps = between(1, 500); for (int i = 0; i < numOps; i++) { long currentMaxSeqNoOfUpdates = engine.getMaxSeqNoOfUpdatesOrDeletes(); - ParsedDocument doc = createParsedDoc(Integer.toString(between(1, 100)), null); + ParsedDocument doc = createParsedDoc(Integer.toString(between(1, 100)), idFieldType, null); if (randomBoolean()) { Engine.IndexResult result = engine.index(indexForDoc(doc)); if (liveDocIds.add(doc.id()) == false) { @@ -6761,7 +6784,7 @@ public void testRefreshAndCloseEngineConcurrently() throws Exception { while (stopped.get() == false) { String id = Integer.toString(randomIntBetween(1, 100)); try { - engine.index(indexForDoc(createParsedDoc(id, null))); + engine.index(indexForDoc(createParsedDoc(id, idFieldType, null))); indexedDocs.release(); } catch (IOException e) { throw new AssertionError(e); @@ -7010,7 +7033,7 @@ private void runTestDeleteFailure(final CheckedBiConsumer new IllegalArgumentException("fatal")); @@ -7024,8 +7047,8 @@ private void runTestDeleteFailure(final CheckedBiConsumer { try { assertTrue(engine.throttleLockIsHeldByCurrentThread()); @@ -7083,7 +7106,7 @@ public void afterRefresh(boolean didRefresh) { Set ids = new HashSet<>(); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); - engine.index(indexForDoc(createParsedDoc(id, null))); + engine.index(indexForDoc(createParsedDoc(id, idFieldType, null))); ids.add(id); } final int refreshCountBeforeGet = refreshCount.get(); @@ -7094,7 +7117,7 @@ public void afterRefresh(boolean didRefresh) { phaser.arriveAndAwaitAdvance(); int iters = randomIntBetween(1, 10); for (int i = 0; i < iters; i++) { - ParsedDocument doc = createParsedDoc(randomFrom(ids), null); + ParsedDocument doc = createParsedDoc(randomFrom(ids), idFieldType, null); try ( Engine.GetResult getResult = engine.get( newGet(true, doc), @@ -7112,7 +7135,7 @@ public void afterRefresh(boolean didRefresh) { } phaser.arriveAndAwaitAdvance(); for (int i = 0; i < numDocs; i++) { - engine.index(indexForDoc(createParsedDoc("more-" + i, null))); + engine.index(indexForDoc(createParsedDoc("more-" + i, idFieldType, null))); } for (Thread getter : getters) { getter.join(); @@ -7156,7 +7179,7 @@ public void afterRefresh(boolean didRefresh) { ); try (InternalEngine engine = createEngine(config)) { if (randomBoolean()) { - engine.index(indexForDoc(createParsedDoc("id", null))); + engine.index(indexForDoc(createParsedDoc("id", idFieldType, null))); } threadPool.executor(ThreadPool.Names.REFRESH) .execute( @@ -7288,7 +7311,7 @@ public void testMaxDocsOnPrimary() throws Exception { for (int i = 0; i < numDocs; i++) { final String id = Integer.toString(randomInt(numDocs)); if (randomBoolean()) { - operations.add(indexForDoc(createParsedDoc(id, null))); + operations.add(indexForDoc(createParsedDoc(id, idFieldType, null))); } else { operations.add(new Engine.Delete(id, newUid(id), primaryTerm.get())); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java index d682f4336d3e2..7a0066520cd80 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java @@ -60,7 +60,7 @@ public void testBasics() throws Exception { int refreshedSeqNo = -1; for (int i = 0; i < numOps; i++) { String id = Integer.toString(randomIntBetween(i, i + 5)); - ParsedDocument doc = createParsedDoc(id, null, randomBoolean()); + ParsedDocument doc = createParsedDoc(id, idFieldType, null, randomBoolean()); if (randomBoolean()) { engine.index(indexForDoc(doc)); } else { @@ -257,7 +257,7 @@ public void testUpdateAndReadChangesConcurrently() throws Exception { int numOps = frequently() ? scaledRandomIntBetween(1, 1500) : scaledRandomIntBetween(5000, 20_000); for (int i = 0; i < numOps; i++) { String id = Integer.toString(randomIntBetween(0, randomBoolean() ? 10 : numOps * 2)); - ParsedDocument doc = createParsedDoc(id, randomAlphaOfLengthBetween(1, 5), randomBoolean()); + ParsedDocument doc = createParsedDoc(id, idFieldType, randomAlphaOfLengthBetween(1, 5), randomBoolean()); final Engine.Operation op; if (onPrimary) { if (randomBoolean()) { @@ -291,14 +291,14 @@ public void testAccessStoredFieldsSequentially() throws Exception { int smallBatch = between(5, 9); long seqNo = 0; for (int i = 0; i < smallBatch; i++) { - engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), null), 1, seqNo, true)); + engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), idFieldType, null), 1, seqNo, true)); seqNo++; } - engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(1000), null), 1, 1000, true)); + engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(1000), idFieldType, null), 1, 1000, true)); seqNo = 11; int largeBatch = between(15, 100); for (int i = 0; i < largeBatch; i++) { - engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), null), 1, seqNo, true)); + engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), idFieldType, null), 1, seqNo, true)); seqNo++; } // disable optimization for a small batch diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java index e1f1a51804b82..b82f9ab79f7df 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -106,7 +106,7 @@ public void testNoOpEngineStats() throws Exception { int deletions = 0; try (InternalEngine engine = createEngine(config)) { for (int i = 0; i < numDocs; i++) { - engine.index(indexForDoc(createParsedDoc(Integer.toString(i), null))); + engine.index(indexForDoc(createParsedDoc(Integer.toString(i), idFieldType, null))); if (rarely()) { engine.flush(); } @@ -166,7 +166,7 @@ public void testTrimUnreferencedTranslogFiles() throws Exception { int totalTranslogOps = 0; for (int i = 0; i < numDocs; i++) { totalTranslogOps++; - engine.index(indexForDoc(createParsedDoc(Integer.toString(i), null))); + engine.index(indexForDoc(createParsedDoc(Integer.toString(i), idFieldType, null))); tracker.updateLocalCheckpoint(allocationId.getId(), i); if (rarely()) { totalTranslogOps = 0; diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java index ce1e7f95d7d19..c4cceaa7d1dbd 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.Term; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.InternalEngineTests; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -126,7 +126,7 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) { indexingOperationListeners, logger ); - ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null); + ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null); Engine.Delete delete = new Engine.Delete("1", new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong()); Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong(), doc); compositeListener.postDelete( diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index ac4e8369855a6..c0fb94c4e9800 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -13,6 +13,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; @@ -83,6 +84,7 @@ import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.seqno.LocalCheckpointTracker; @@ -163,6 +165,11 @@ public abstract class EngineTestCase extends ESTestCase { protected Path replicaTranslogDir; // A default primary term is used by engine instances created in this test. protected final PrimaryTermSupplier primaryTerm = new PrimaryTermSupplier(1L); + /** + * {@link FieldType} for a random {@link IdFieldMapper} chosen at the start of the + * test. + */ + protected FieldType idFieldType; protected static void assertVisibleCount(Engine engine, int numDocs) throws IOException { assertVisibleCount(engine, numDocs, true); @@ -232,6 +239,14 @@ public void setUp() throws Exception { if (randomBoolean()) { engine.config().setEnableGcDeletes(false); } + idFieldType = randomIdFieldType(); + } + + /** + * Chose a random {@link FieldType} from the list of all {@link IdFieldMapper}s. + */ + public static FieldType randomIdFieldType() { + return randomBoolean() ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : TsidExtractingIdFieldMapper.FIELD_TYPE; } public EngineConfig copy(EngineConfig config, LongSupplier globalCheckpointSupplier) { @@ -356,13 +371,22 @@ protected static LuceneDocument testDocument() { return new LuceneDocument(); } - public static ParsedDocument createParsedDoc(String id, String routing) { - return testParsedDocument(id, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null); + public static ParsedDocument createParsedDoc(String id, FieldType idFieldType, String routing) { + return testParsedDocument( + id, + idFieldType, + routing, + testDocumentWithTextField(), + new BytesArray("{ \"value\" : \"test\" }"), + null, + false + ); } - public static ParsedDocument createParsedDoc(String id, String routing, boolean recoverySource) { + public static ParsedDocument createParsedDoc(String id, FieldType idFieldType, String routing, boolean recoverySource) { return testParsedDocument( id, + idFieldType, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), @@ -371,28 +395,29 @@ public static ParsedDocument createParsedDoc(String id, String routing, boolean ); } - protected static ParsedDocument testParsedDocument( + protected ParsedDocument testParsedDocument( String id, String routing, LuceneDocument document, BytesReference source, Mapping mappingUpdate ) { - return testParsedDocument(id, routing, document, source, mappingUpdate, false); + return testParsedDocument(id, idFieldType, routing, document, source, mappingUpdate, false); } protected static ParsedDocument testParsedDocument( String id, + FieldType idFieldType, String routing, LuceneDocument document, BytesReference source, Mapping mappingUpdate, boolean recoverySource - ) { // TODO try with TsdbIdFieldMapper - Field uidField = new Field("_id", Uid.encodeId(id), ProvidedIdFieldMapper.Defaults.FIELD_TYPE); + ) { + Field idField = new Field("_id", Uid.encodeId(id), idFieldType); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); - document.add(uidField); + document.add(idField); document.add(versionField); document.add(seqID.seqNo); document.add(seqID.seqNoDocValue); @@ -958,6 +983,7 @@ public static List generateSingleDocHistory( final int startWithSeqNo = 0; final String valuePrefix = (forReplica ? "r_" : "p_") + docId + "_"; final boolean incrementTermWhenIntroducingSeqNo = randomBoolean(); + FieldType idFieldType = randomIdFieldType(); for (int i = 0; i < numOfOps; i++) { final Engine.Operation op; final long version = switch (versionType) { @@ -968,7 +994,7 @@ public static List generateSingleDocHistory( if (randomBoolean()) { op = new Engine.Index( id, - testParsedDocument(docId, null, testDocumentWithTextField(valuePrefix + i), SOURCE, null), + testParsedDocument(docId, idFieldType, null, testDocumentWithTextField(valuePrefix + i), SOURCE, null, false), forReplica && i >= startWithSeqNo ? i * 2 : SequenceNumbers.UNASSIGNED_SEQ_NO, forReplica && i >= startWithSeqNo && incrementTermWhenIntroducingSeqNo ? primaryTerm + 1 : primaryTerm, version, @@ -1018,6 +1044,7 @@ public List generateHistoryOnReplica( long seqNo = startingSeqNo; final int maxIdValue = randomInt(numOps * 2); final List operations = new ArrayList<>(numOps); + FieldType idFieldType = includeNestedDocs ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : randomIdFieldType(); CheckedBiFunction nestedParsedDocFactory = nestedParsedDocFactory(); for (int i = 0; i < numOps; i++) { final String id = Integer.toString(randomInt(maxIdValue)); @@ -1027,7 +1054,9 @@ public List generateHistoryOnReplica( final long startTime = threadPool.relativeTimeInNanos(); final int copies = allowDuplicate && rarely() ? between(2, 4) : 1; for (int copy = 0; copy < copies; copy++) { - final ParsedDocument doc = isNestedDoc ? nestedParsedDocFactory.apply(id, nestedValues) : createParsedDoc(id, null); + final ParsedDocument doc = isNestedDoc + ? nestedParsedDocFactory.apply(id, nestedValues) + : createParsedDoc(id, idFieldType, null); switch (opType) { case INDEX -> operations.add( new Engine.Index( diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 5886fd92eace4..446ff71171031 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.ccr.index.engine; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; @@ -20,6 +21,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -31,6 +33,8 @@ import org.elasticsearch.index.engine.InternalEngine; import org.elasticsearch.index.engine.TranslogHandler; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShard; @@ -75,15 +79,30 @@ public class FollowingEngineTests extends ESTestCase { private ShardId shardId; private AtomicLong primaryTerm = new AtomicLong(); private AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + private IndexMode indexMode; + private FieldType idFieldType; + @Override public void setUp() throws Exception { super.setUp(); threadPool = new TestThreadPool("following-engine-tests"); index = new Index("index", "uuid"); shardId = new ShardId(index, 0); primaryTerm.set(randomLongBetween(1, Long.MAX_VALUE)); + indexMode = randomFrom(IndexMode.values()); + switch (indexMode) { + case STANDARD: + idFieldType = ProvidedIdFieldMapper.Defaults.FIELD_TYPE; + break; + case TIME_SERIES: + idFieldType = TsidExtractingIdFieldMapper.FIELD_TYPE; + break; + default: + throw new UnsupportedOperationException("Unknown index mode [" + indexMode + "]"); + } } + @Override public void tearDown() throws Exception { terminate(threadPool); super.tearDown(); @@ -296,7 +315,7 @@ private Engine.Index indexForFollowing(String id, long seqNo, Engine.Operation.O } private Engine.Index indexForFollowing(String id, long seqNo, Engine.Operation.Origin origin, long version) { - final ParsedDocument parsedDocument = EngineTestCase.createParsedDoc(id, null); + final ParsedDocument parsedDocument = EngineTestCase.createParsedDoc(id, idFieldType, null); return new Engine.Index( EngineTestCase.newUid(parsedDocument), parsedDocument, @@ -327,12 +346,12 @@ private Engine.Delete deleteForFollowing(String id, long seqNo, Engine.Operation } private Engine.Index indexForPrimary(String id) { - final ParsedDocument parsedDoc = EngineTestCase.createParsedDoc(id, null); + final ParsedDocument parsedDoc = EngineTestCase.createParsedDoc(id, idFieldType, null); return new Engine.Index(EngineTestCase.newUid(parsedDoc), primaryTerm.get(), parsedDoc); } private Engine.Delete deleteForPrimary(String id) { - final ParsedDocument parsedDoc = EngineTestCase.createParsedDoc(id, null); + final ParsedDocument parsedDoc = EngineTestCase.createParsedDoc(id, idFieldType, null); return new Engine.Delete(parsedDoc.id(), EngineTestCase.newUid(parsedDoc), primaryTerm.get()); } @@ -717,12 +736,21 @@ public void close() throws IOException { } public void testProcessOnceOnPrimary() throws Exception { - final Settings settings = Settings.builder() + final Settings.Builder settingsBuilder = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .put("index.version.created", Version.CURRENT) - .put("index.xpack.ccr.following_index", true) - .build(); + .put("index.xpack.ccr.following_index", true); + switch (indexMode) { + case STANDARD: + break; + case TIME_SERIES: + settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo"); + break; + default: + throw new UnsupportedOperationException("Unknown index mode [" + indexMode + "]"); + } + final Settings settings = settingsBuilder.build(); final IndexMetadata indexMetadata = IndexMetadata.builder(index.getName()).settings(settings).build(); final IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); final CheckedBiFunction nestedDocFunc = EngineTestCase.nestedParsedDocFactory(); @@ -730,7 +758,9 @@ public void testProcessOnceOnPrimary() throws Exception { List operations = new ArrayList<>(numOps); for (int i = 0; i < numOps; i++) { String docId = Integer.toString(between(1, 100)); - ParsedDocument doc = randomBoolean() ? EngineTestCase.createParsedDoc(docId, null) : nestedDocFunc.apply(docId, randomInt(3)); + ParsedDocument doc = randomBoolean() + ? EngineTestCase.createParsedDoc(docId, idFieldType, null) + : nestedDocFunc.apply(docId, randomInt(3)); if (randomBoolean()) { operations.add( new Engine.Index(