Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use tsdb's id in Engine tests #85055

Merged
merged 2 commits into from
Mar 21, 2022
Merged

Use tsdb's id in Engine tests #85055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
517b72d
Use tsdb's id in Engine tests
nik9000 Mar 17, 2022
2d1592d
same field type for all docs in a test
nik9000 Mar 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
* {@code _tsid} and {@code @timestamp}.
*/
public class TsidExtractingIdFieldMapper extends IdFieldMapper {
private static final FieldType FIELD_TYPE = new FieldType();
public static final FieldType FIELD_TYPE = new FieldType();
/**
* Maximum length of the {@code _tsid} in the {@link #documentDescription}.
*/
Expand Down
4 changes: 2 additions & 2 deletions server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@
import org.elasticsearch.index.cache.query.IndexQueryCache;
import org.elasticsearch.index.cache.query.QueryCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineTestCase;
import org.elasticsearch.index.engine.InternalEngineFactory;
import org.elasticsearch.index.engine.InternalEngineTests;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.ParsedDocument;
Expand Down Expand Up @@ -306,7 +306,7 @@ public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
assertEquals(IndexingSlowLog.class, indexService.getIndexOperationListeners().get(0).getClass());
assertSame(listener, indexService.getIndexOperationListeners().get(1));

ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null);
ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null);
Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), randomNonNegativeLong(), doc);
ShardId shardId = new ShardId(new Index("foo", "bar"), 0);
for (IndexingOperationListener l : indexService.getIndexOperationListeners()) {
Expand Down
6 changes: 3 additions & 3 deletions server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.IndexingSlowLog.IndexingSlowLogMessage;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.InternalEngineTests;
import org.elasticsearch.index.engine.EngineTestCase;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SeqNoFieldMapper;
import org.elasticsearch.index.mapper.Uid;
Expand Down Expand Up @@ -74,7 +74,7 @@ public void testLevelPrecedence() {
IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
IndexingSlowLog log = new IndexingSlowLog(settings);

ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null);
ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null);
Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId("doc_id")), randomNonNegativeLong(), doc);
Engine.IndexResult result = Mockito.mock(Engine.IndexResult.class);// (0, 0, SequenceNumbers.UNASSIGNED_SEQ_NO, false);
Mockito.when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS);
Expand Down Expand Up @@ -148,7 +148,7 @@ public void testTwoLoggersDifferentLevel() {
);
IndexingSlowLog log2 = new IndexingSlowLog(index2Settings);

ParsedDocument doc = InternalEngineTests.createParsedDoc("1", null);
ParsedDocument doc = EngineTestCase.createParsedDoc("1", EngineTestCase.randomIdFieldType(), null);
Engine.Index index = new Engine.Index(new Term("_id", Uid.encodeId("doc_id")), randomNonNegativeLong(), doc);
Engine.IndexResult result = Mockito.mock(Engine.IndexResult.class);
Mockito.when(result.getResultType()).thenReturn(Engine.Result.Type.SUCCESS);
Expand Down
65 changes: 44 additions & 21 deletions server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.filter.RegexFilter;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
Expand Down Expand Up @@ -107,6 +108,7 @@
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
import org.elasticsearch.index.mapper.SeqNoFieldMapper;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.VersionFieldMapper;
import org.elasticsearch.index.seqno.LocalCheckpointTracker;
Expand Down Expand Up @@ -985,7 +987,7 @@ public void testSimpleOperations() throws Exception {

public void testGetWithSearcherWrapper() throws Exception {
engine.refresh("warm_up");
engine.index(indexForDoc(createParsedDoc("1", null)));
engine.index(indexForDoc(createParsedDoc("1", idFieldType, null)));
assertThat(engine.lastRefreshedCheckpoint(), equalTo(NO_OPS_PERFORMED));
MapperService mapperService = createMapperService();
MappingLookup mappingLookup = mapperService.mappingLookup();
Expand All @@ -1003,7 +1005,7 @@ public void testGetWithSearcherWrapper() throws Exception {
// refresh triggered, as we did not track translog location until the first realtime get.
assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L));

engine.index(indexForDoc(createParsedDoc("1", null)));
engine.index(indexForDoc(createParsedDoc("1", idFieldType, null)));
try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), mappingLookup, documentParser, searcher -> searcher)) {
assertTrue(get.exists());
assertEquals(++translogGetCountExpected, translogGetCount.getAsLong());
Expand All @@ -1012,7 +1014,7 @@ public void testGetWithSearcherWrapper() throws Exception {
assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L)); // no refresh; just read from translog

if (randomBoolean()) {
engine.index(indexForDoc(createParsedDoc("1", null)));
engine.index(indexForDoc(createParsedDoc("1", idFieldType, null)));
}
try (
Engine.GetResult get = engine.get(
Expand Down Expand Up @@ -1069,7 +1071,7 @@ public void testGetWithSearcherWrapper() throws Exception {
}
assertThat("no refresh, just read from translog or in-memory segment", engine.lastRefreshedCheckpoint(), equalTo(0L));

engine.index(indexForDoc(createParsedDoc("1", null)));
engine.index(indexForDoc(createParsedDoc("1", idFieldType, null)));
try (
Engine.GetResult get = engine.get(
new Engine.Get(true, true, "1"),
Expand Down Expand Up @@ -1517,6 +1519,7 @@ public void testVersioningNewIndex() throws IOException {
* we are testing an edge case here where we have a fully deleted segment that is retained but has all it's IDs pruned away.
*/
public void testLookupVersionWithPrunedAwayIds() throws IOException {
FieldType idFieldType = randomBoolean() ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : TsidExtractingIdFieldMapper.FIELD_TYPE;
try (Directory dir = newDirectory()) {
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Lucene.STANDARD_ANALYZER);
indexWriterConfig.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD);
Expand All @@ -1533,7 +1536,7 @@ public void testLookupVersionWithPrunedAwayIds() throws IOException {
)
) {
org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document();
doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE));
doc.add(new Field(IdFieldMapper.NAME, "1", idFieldType));
doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, -1));
doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 1));
doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1));
Expand Down Expand Up @@ -1668,7 +1671,15 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
int numDocs = scaledRandomIntBetween(10, 100);
for (int i = 0; i < numDocs; i++) {
boolean useRecoverySource = randomBoolean() || omitSourceAllTheTime;
ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null, useRecoverySource);
ParsedDocument doc = testParsedDocument(
Integer.toString(i),
idFieldType,
null,
testDocument(),
B_1,
null,
useRecoverySource
);
engine.index(indexForDoc(doc));
liveDocs.add(doc.id());
if (useRecoverySource == false) {
Expand All @@ -1677,7 +1688,15 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
}
for (int i = 0; i < numDocs; i++) {
boolean useRecoverySource = randomBoolean() || omitSourceAllTheTime;
ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null, useRecoverySource);
ParsedDocument doc = testParsedDocument(
Integer.toString(i),
idFieldType,
null,
testDocument(),
B_1,
null,
useRecoverySource
);
if (randomBoolean()) {
engine.delete(new Engine.Delete(doc.id(), newUid(doc.id()), primaryTerm.get()));
liveDocs.remove(doc.id());
Expand Down Expand Up @@ -1737,7 +1756,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
}
if (numSegments == 1) {
boolean useRecoverySource = randomBoolean() || omitSourceAllTheTime;
ParsedDocument doc = testParsedDocument("dummy", null, testDocument(), B_1, null, useRecoverySource);
ParsedDocument doc = testParsedDocument("dummy", idFieldType, null, testDocument(), B_1, null, useRecoverySource);
engine.index(indexForDoc(doc));
if (useRecoverySource == false) {
liveDocsWithSource.add(doc.id());
Expand Down Expand Up @@ -4626,7 +4645,7 @@ public void testLookupSeqNoByIdInLucene() throws Exception {
boolean isIndexing = randomBoolean();
int copies = frequently() ? 1 : between(2, 4);
for (int c = 0; c < copies; c++) {
final ParsedDocument doc = EngineTestCase.createParsedDoc(id, null);
final ParsedDocument doc = EngineTestCase.createParsedDoc(id, idFieldType, null);
if (isIndexing) {
operations.add(
new Engine.Index(
Expand Down Expand Up @@ -5067,7 +5086,7 @@ public void testRandomOperations() throws Exception {
int numOps = between(10, 100);
for (int i = 0; i < numOps; i++) {
String id = Integer.toString(randomIntBetween(1, 10));
ParsedDocument doc = createParsedDoc(id, null);
ParsedDocument doc = createParsedDoc(id, idFieldType, null);
Engine.Operation.TYPE type = randomFrom(Engine.Operation.TYPE.values());
switch (type) {
case INDEX -> {
Expand Down Expand Up @@ -5467,7 +5486,11 @@ public void testSeqNoGenerator() throws IOException {
)
) {
final String id = "id";
final Field uidField = new Field("_id", id, ProvidedIdFieldMapper.Defaults.FIELD_TYPE);
final Field uidField = new Field(
"_id",
id,
randomBoolean() ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : TsidExtractingIdFieldMapper.FIELD_TYPE
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm thinking I should pick this up front rather than per document....

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll do that now.

);
final Field versionField = new NumericDocValuesField("_version", 0);
final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
final LuceneDocument document = new LuceneDocument();
Expand Down Expand Up @@ -6454,7 +6477,7 @@ public void testTrackMaxSeqNoOfUpdatesOrDeletesOnPrimary() throws Exception {
int numOps = between(1, 500);
for (int i = 0; i < numOps; i++) {
long currentMaxSeqNoOfUpdates = engine.getMaxSeqNoOfUpdatesOrDeletes();
ParsedDocument doc = createParsedDoc(Integer.toString(between(1, 100)), null);
ParsedDocument doc = createParsedDoc(Integer.toString(between(1, 100)), idFieldType, null);
if (randomBoolean()) {
Engine.IndexResult result = engine.index(indexForDoc(doc));
if (liveDocIds.add(doc.id()) == false) {
Expand Down Expand Up @@ -6761,7 +6784,7 @@ public void testRefreshAndCloseEngineConcurrently() throws Exception {
while (stopped.get() == false) {
String id = Integer.toString(randomIntBetween(1, 100));
try {
engine.index(indexForDoc(createParsedDoc(id, null)));
engine.index(indexForDoc(createParsedDoc(id, idFieldType, null)));
indexedDocs.release();
} catch (IOException e) {
throw new AssertionError(e);
Expand Down Expand Up @@ -7010,7 +7033,7 @@ private void runTestDeleteFailure(final CheckedBiConsumer<InternalEngine, Engine
iw.set(new ThrowingIndexWriter(dir, iwc));
return iw.get();
}, null, null, config(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
engine.index(new Engine.Index(newUid("0"), primaryTerm.get(), InternalEngineTests.createParsedDoc("0", null)));
engine.index(new Engine.Index(newUid("0"), primaryTerm.get(), InternalEngineTests.createParsedDoc("0", idFieldType, null)));
final Engine.Delete op = new Engine.Delete("0", newUid("0"), primaryTerm.get());
consumer.accept(engine, op);
iw.get().setThrowFailure(() -> new IllegalArgumentException("fatal"));
Expand All @@ -7024,8 +7047,8 @@ private void runTestDeleteFailure(final CheckedBiConsumer<InternalEngine, Engine
}

public void testIndexThrottling() throws Exception {
final Engine.Index indexWithThrottlingCheck = spy(indexForDoc(createParsedDoc("1", null)));
final Engine.Index indexWithoutThrottlingCheck = spy(indexForDoc(createParsedDoc("2", null)));
final Engine.Index indexWithThrottlingCheck = spy(indexForDoc(createParsedDoc("1", idFieldType, null)));
final Engine.Index indexWithoutThrottlingCheck = spy(indexForDoc(createParsedDoc("2", idFieldType, null)));
doAnswer(invocation -> {
try {
assertTrue(engine.throttleLockIsHeldByCurrentThread());
Expand Down Expand Up @@ -7083,7 +7106,7 @@ public void afterRefresh(boolean didRefresh) {
Set<String> ids = new HashSet<>();
for (int i = 0; i < numDocs; i++) {
String id = Integer.toString(i);
engine.index(indexForDoc(createParsedDoc(id, null)));
engine.index(indexForDoc(createParsedDoc(id, idFieldType, null)));
ids.add(id);
}
final int refreshCountBeforeGet = refreshCount.get();
Expand All @@ -7094,7 +7117,7 @@ public void afterRefresh(boolean didRefresh) {
phaser.arriveAndAwaitAdvance();
int iters = randomIntBetween(1, 10);
for (int i = 0; i < iters; i++) {
ParsedDocument doc = createParsedDoc(randomFrom(ids), null);
ParsedDocument doc = createParsedDoc(randomFrom(ids), idFieldType, null);
try (
Engine.GetResult getResult = engine.get(
newGet(true, doc),
Expand All @@ -7112,7 +7135,7 @@ public void afterRefresh(boolean didRefresh) {
}
phaser.arriveAndAwaitAdvance();
for (int i = 0; i < numDocs; i++) {
engine.index(indexForDoc(createParsedDoc("more-" + i, null)));
engine.index(indexForDoc(createParsedDoc("more-" + i, idFieldType, null)));
}
for (Thread getter : getters) {
getter.join();
Expand Down Expand Up @@ -7156,7 +7179,7 @@ public void afterRefresh(boolean didRefresh) {
);
try (InternalEngine engine = createEngine(config)) {
if (randomBoolean()) {
engine.index(indexForDoc(createParsedDoc("id", null)));
engine.index(indexForDoc(createParsedDoc("id", idFieldType, null)));
}
threadPool.executor(ThreadPool.Names.REFRESH)
.execute(
Expand Down Expand Up @@ -7288,7 +7311,7 @@ public void testMaxDocsOnPrimary() throws Exception {
for (int i = 0; i < numDocs; i++) {
final String id = Integer.toString(randomInt(numDocs));
if (randomBoolean()) {
operations.add(indexForDoc(createParsedDoc(id, null)));
operations.add(indexForDoc(createParsedDoc(id, idFieldType, null)));
} else {
operations.add(new Engine.Delete(id, newUid(id), primaryTerm.get()));
}
Expand Down
10 changes: 5 additions & 5 deletions server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public void testBasics() throws Exception {
int refreshedSeqNo = -1;
for (int i = 0; i < numOps; i++) {
String id = Integer.toString(randomIntBetween(i, i + 5));
ParsedDocument doc = createParsedDoc(id, null, randomBoolean());
ParsedDocument doc = createParsedDoc(id, idFieldType, null, randomBoolean());
if (randomBoolean()) {
engine.index(indexForDoc(doc));
} else {
Expand Down Expand Up @@ -257,7 +257,7 @@ public void testUpdateAndReadChangesConcurrently() throws Exception {
int numOps = frequently() ? scaledRandomIntBetween(1, 1500) : scaledRandomIntBetween(5000, 20_000);
for (int i = 0; i < numOps; i++) {
String id = Integer.toString(randomIntBetween(0, randomBoolean() ? 10 : numOps * 2));
ParsedDocument doc = createParsedDoc(id, randomAlphaOfLengthBetween(1, 5), randomBoolean());
ParsedDocument doc = createParsedDoc(id, idFieldType, randomAlphaOfLengthBetween(1, 5), randomBoolean());
final Engine.Operation op;
if (onPrimary) {
if (randomBoolean()) {
Expand Down Expand Up @@ -291,14 +291,14 @@ public void testAccessStoredFieldsSequentially() throws Exception {
int smallBatch = between(5, 9);
long seqNo = 0;
for (int i = 0; i < smallBatch; i++) {
engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), null), 1, seqNo, true));
engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), idFieldType, null), 1, seqNo, true));
seqNo++;
}
engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(1000), null), 1, 1000, true));
engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(1000), idFieldType, null), 1, 1000, true));
seqNo = 11;
int largeBatch = between(15, 100);
for (int i = 0; i < largeBatch; i++) {
engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), null), 1, seqNo, true));
engine.index(replicaIndexForDoc(createParsedDoc(Long.toString(seqNo), idFieldType, null), 1, seqNo, true));
seqNo++;
}
// disable optimization for a small batch
Expand Down
Loading