Skip to content

Commit 4b29d6b

Browse files
kkewweikkewwei
committed
Modify numeric FieldMappers to enable SkipList
1 parent 9a3fc30 commit 4b29d6b

File tree

70 files changed

+488
-487
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+488
-487
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1414
### Changed
1515
- Change the default max header size from 8KB to 16KB. ([#18024](https://github.com/opensearch-project/OpenSearch/pull/18024))
1616
- Avoid invalid retries in multiple replicas when querying [#17370](https://github.com/opensearch-project/OpenSearch/pull/17370)
17+
- Modify numeric FieldMappers to enable SkipList ([#18066](https://github.com/opensearch-project/OpenSearch/pull/18066))
1718

1819
### Dependencies
1920
- Bump `com.google.code.gson:gson` from 2.12.1 to 2.13.0 ([#17923](https://github.com/opensearch-project/OpenSearch/pull/17923))

‎modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,10 +105,10 @@ public void testTwoFields() throws Exception {
105105
for (int docId = 0; docId < numDocs; docId++) {
106106
Document document = new Document();
107107
fieldAValues[docId] = randomDouble();
108-
document.add(new SortedNumericDocValuesField(fieldA, NumericUtils.doubleToSortableLong(fieldAValues[docId])));
108+
document.add(SortedNumericDocValuesField.indexedField(fieldA, NumericUtils.doubleToSortableLong(fieldAValues[docId])));
109109

110110
fieldBValues[docId] = randomDouble();
111-
document.add(new SortedNumericDocValuesField(fieldB, NumericUtils.doubleToSortableLong(fieldBValues[docId])));
111+
document.add(SortedNumericDocValuesField.indexedField(fieldB, NumericUtils.doubleToSortableLong(fieldBValues[docId])));
112112
indexWriter.addDocument(document);
113113
}
114114

‎modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -186,9 +186,9 @@ public void testFieldData() throws IOException {
186186
Directory dir = newDirectory();
187187
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
188188
Document doc = new Document();
189-
doc.add(new SortedNumericDocValuesField("scaled_float1", 10));
190-
doc.add(new SortedNumericDocValuesField("scaled_float2", 5));
191-
doc.add(new SortedNumericDocValuesField("scaled_float2", 12));
189+
doc.add(SortedNumericDocValuesField.indexedField("scaled_float1", 10));
190+
doc.add(SortedNumericDocValuesField.indexedField("scaled_float2", 5));
191+
doc.add(SortedNumericDocValuesField.indexedField("scaled_float2", 12));
192192
w.addDocument(doc);
193193
try (DirectoryReader reader = DirectoryReader.open(w)) {
194194
// single-valued

‎modules/parent-join/src/test/java/org/opensearch/join/aggregations/ChildrenToParentAggregatorTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ private static List<Field> createParentDocument(String id, int value) {
274274
new StringField(IdFieldMapper.NAME, Uid.encodeId(id), Field.Store.NO),
275275
new StringField("join_field", PARENT_TYPE, Field.Store.NO),
276276
createJoinField(PARENT_TYPE, id),
277-
new SortedNumericDocValuesField("number", value)
277+
SortedNumericDocValuesField.indexedField("number", value)
278278
);
279279
}
280280

@@ -283,7 +283,7 @@ private static List<Field> createChildDocument(String childId, String parentId,
283283
new StringField(IdFieldMapper.NAME, Uid.encodeId(childId), Field.Store.NO),
284284
new StringField("join_field", CHILD_TYPE, Field.Store.NO),
285285
createJoinField(PARENT_TYPE, parentId),
286-
new SortedNumericDocValuesField("subNumber", value)
286+
SortedNumericDocValuesField.indexedField("subNumber", value)
287287
);
288288
}
289289

‎modules/parent-join/src/test/java/org/opensearch/join/aggregations/ParentToChildrenAggregatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ private static List<Field> createChildDocument(String childId, String parentId,
241241
new StringField(IdFieldMapper.NAME, Uid.encodeId(childId), Field.Store.NO),
242242
new StringField("join_field", CHILD_TYPE, Field.Store.NO),
243243
createJoinField(PARENT_TYPE, parentId),
244-
new SortedNumericDocValuesField("number", value)
244+
SortedNumericDocValuesField.indexedField("number", value)
245245
);
246246
}
247247

‎server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -778,7 +778,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
778778
context.doc().add(new LongPoint(fieldType().name(), timestamp));
779779
}
780780
if (hasDocValues) {
781-
context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp));
781+
context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp));
782782
} else if (store || indexed) {
783783
createFieldNamesField(context);
784784
}

‎server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
324324
fields.add(new HalfFloatPoint(name, value.floatValue()));
325325
}
326326
if (docValued) {
327-
fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
327+
fields.add(SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
328328
}
329329
if (stored) {
330330
fields.add(new StoredField(name, value.floatValue()));
@@ -471,7 +471,7 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
471471
fields.add(new FloatPoint(name, value.floatValue()));
472472
}
473473
if (docValued) {
474-
fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
474+
fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(value.floatValue())));
475475
}
476476
if (stored) {
477477
fields.add(new StoredField(name, value.floatValue()));
@@ -597,7 +597,7 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
597597
fields.add(new DoublePoint(name, value.doubleValue()));
598598
}
599599
if (docValued) {
600-
fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
600+
fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
601601
}
602602
if (stored) {
603603
fields.add(new StoredField(name, value.doubleValue()));
@@ -956,7 +956,7 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
956956
fields.add(new IntPoint(name, value.intValue()));
957957
}
958958
if (docValued) {
959-
fields.add(new SortedNumericDocValuesField(name, value.intValue()));
959+
fields.add(SortedNumericDocValuesField.indexedField(name, value.intValue()));
960960
}
961961
if (stored) {
962962
fields.add(new StoredField(name, value.intValue()));
@@ -1099,7 +1099,7 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
10991099
fields.add(new LongPoint(name, value.longValue()));
11001100
}
11011101
if (docValued) {
1102-
fields.add(new SortedNumericDocValuesField(name, value.longValue()));
1102+
fields.add(SortedNumericDocValuesField.indexedField(name, value.longValue()));
11031103
}
11041104
if (stored) {
11051105
fields.add(new StoredField(name, value.longValue()));
@@ -1221,7 +1221,7 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
12211221
}
12221222

12231223
if (docValued) {
1224-
fields.add(new SortedNumericDocValuesField(name, v.longValue()));
1224+
fields.add(SortedNumericDocValuesField.indexedField(name, v.longValue()));
12251225
}
12261226

12271227
if (stored) {

‎server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -57,29 +57,29 @@ public void testStarTreeDocValues() throws IOException {
5757
conf.setMergePolicy(newLogMergePolicy());
5858
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
5959
Document doc = new Document();
60-
doc.add(new SortedNumericDocValuesField("unsignedLongDimension", 10));
61-
doc.add(new SortedNumericDocValuesField("sndv", 1));
62-
doc.add(new SortedNumericDocValuesField("dv1", 1));
63-
doc.add(new SortedNumericDocValuesField("field1", -1));
60+
doc.add(SortedNumericDocValuesField.indexedField("unsignedLongDimension", 10));
61+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
62+
doc.add(SortedNumericDocValuesField.indexedField("dv1", 1));
63+
doc.add(SortedNumericDocValuesField.indexedField("field1", -1));
6464
iw.addDocument(doc);
6565
doc = new Document();
66-
doc.add(new SortedNumericDocValuesField("unsignedLongDimension", 10));
67-
doc.add(new SortedNumericDocValuesField("sndv", 1));
68-
doc.add(new SortedNumericDocValuesField("dv1", 1));
69-
doc.add(new SortedNumericDocValuesField("field1", -1));
66+
doc.add(SortedNumericDocValuesField.indexedField("unsignedLongDimension", 10));
67+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
68+
doc.add(SortedNumericDocValuesField.indexedField("dv1", 1));
69+
doc.add(SortedNumericDocValuesField.indexedField("field1", -1));
7070
iw.addDocument(doc);
7171
doc = new Document();
7272
iw.forceMerge(1);
73-
doc.add(new SortedNumericDocValuesField("unsignedLongDimension", -20));
74-
doc.add(new SortedNumericDocValuesField("sndv", 2));
75-
doc.add(new SortedNumericDocValuesField("dv1", 2));
76-
doc.add(new SortedNumericDocValuesField("field1", -2));
73+
doc.add(SortedNumericDocValuesField.indexedField("unsignedLongDimension", -20));
74+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
75+
doc.add(SortedNumericDocValuesField.indexedField("dv1", 2));
76+
doc.add(SortedNumericDocValuesField.indexedField("field1", -2));
7777
iw.addDocument(doc);
7878
doc = new Document();
79-
doc.add(new SortedNumericDocValuesField("unsignedLongDimension", -20));
80-
doc.add(new SortedNumericDocValuesField("sndv", 2));
81-
doc.add(new SortedNumericDocValuesField("dv1", 2));
82-
doc.add(new SortedNumericDocValuesField("field1", -2));
79+
doc.add(SortedNumericDocValuesField.indexedField("unsignedLongDimension", -20));
80+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
81+
doc.add(SortedNumericDocValuesField.indexedField("dv1", 2));
82+
doc.add(SortedNumericDocValuesField.indexedField("field1", -2));
8383
iw.addDocument(doc);
8484
iw.forceMerge(1);
8585
iw.close();
@@ -169,14 +169,14 @@ public void testStarTreeDocValuesWithDeletions() throws IOException {
169169
Document doc = new Document();
170170
doc.add(new StringField("_id", id, Field.Store.YES));
171171
int fieldValue = random().nextInt(5) + 1;
172-
doc.add(new SortedNumericDocValuesField("field", fieldValue));
172+
doc.add(SortedNumericDocValuesField.indexedField("field", fieldValue));
173173

174174
int sndvValue = random().nextInt(3);
175175

176-
doc.add(new SortedNumericDocValuesField("sndv", sndvValue));
176+
doc.add(SortedNumericDocValuesField.indexedField("sndv", sndvValue));
177177
int dvValue = random().nextInt(3);
178178

179-
doc.add(new SortedNumericDocValuesField("dv", dvValue));
179+
doc.add(SortedNumericDocValuesField.indexedField("dv", dvValue));
180180
map.put(sndvValue + "-" + dvValue, fieldValue + map.getOrDefault(sndvValue + "-" + dvValue, 0));
181181
doc.add(new NumericDocValuesField("field-ndv", fieldValue));
182182

‎server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeKeywordDocValuesFormatTests.java‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,14 @@ public void testStarTreeKeywordDocValues() throws IOException {
6666
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
6767
Document doc = new Document();
6868
doc.add(new StringField("_id", "1", Field.Store.NO));
69-
doc.add(new SortedNumericDocValuesField("sndv", 1));
69+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
7070
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
7171
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
7272
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
7373
iw.addDocument(doc);
7474
doc = new Document();
7575
doc.add(new StringField("_id", "2", Field.Store.NO));
76-
doc.add(new SortedNumericDocValuesField("sndv", 1));
76+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
7777
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
7878
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
7979
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
@@ -84,14 +84,14 @@ public void testStarTreeKeywordDocValues() throws IOException {
8484
iw.flush();
8585
doc = new Document();
8686
doc.add(new StringField("_id", "3", Field.Store.NO));
87-
doc.add(new SortedNumericDocValuesField("sndv", 2));
87+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
8888
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
8989
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
9090
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
9191
iw.addDocument(doc);
9292
doc = new Document();
9393
doc.add(new StringField("_id", "4", Field.Store.NO));
94-
doc.add(new SortedNumericDocValuesField("sndv", 2));
94+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
9595
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
9696
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
9797
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
@@ -166,7 +166,7 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {
166166
Document doc = new Document();
167167
doc.add(new StringField("_id", id, Field.Store.YES));
168168
int sndvValue = random().nextInt(5) + 1;
169-
doc.add(new SortedNumericDocValuesField("sndv", sndvValue));
169+
doc.add(SortedNumericDocValuesField.indexedField("sndv", sndvValue));
170170

171171
String keyword1Value = "text" + random().nextInt(3);
172172

@@ -266,26 +266,26 @@ public void testStarKeywordDocValuesWithMissingDocs() throws IOException {
266266
conf.setMergePolicy(newLogMergePolicy());
267267
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
268268
Document doc = new Document();
269-
doc.add(new SortedNumericDocValuesField("sndv", 1));
269+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
270270
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
271271
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
272272

273273
iw.addDocument(doc);
274274
doc = new Document();
275-
doc.add(new SortedNumericDocValuesField("sndv", 1));
275+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
276276
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
277277
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
278278
iw.addDocument(doc);
279279
iw.forceMerge(1);
280280
doc = new Document();
281-
doc.add(new SortedNumericDocValuesField("sndv", 2));
281+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
282282
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
283283
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
284284
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
285285

286286
iw.addDocument(doc);
287287
doc = new Document();
288-
doc.add(new SortedNumericDocValuesField("sndv", 2));
288+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
289289
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
290290
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
291291
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
@@ -351,20 +351,20 @@ public void testStarKeywordDocValuesWithMissingDocsInSegment() throws IOExceptio
351351
conf.setMergePolicy(newLogMergePolicy());
352352
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
353353
Document doc = new Document();
354-
doc.add(new SortedNumericDocValuesField("sndv", 1));
354+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
355355
iw.addDocument(doc);
356356
doc = new Document();
357-
doc.add(new SortedNumericDocValuesField("sndv", 1));
357+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
358358
iw.addDocument(doc);
359359
iw.forceMerge(1);
360360
doc = new Document();
361-
doc.add(new SortedNumericDocValuesField("sndv", 2));
361+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
362362
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
363363
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
364364
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
365365
iw.addDocument(doc);
366366
doc = new Document();
367-
doc.add(new SortedNumericDocValuesField("sndv", 2));
367+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
368368
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
369369
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
370370
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
@@ -428,17 +428,17 @@ public void testStarKeywordDocValuesWithMissingDocsInAllSegments() throws IOExce
428428
conf.setMergePolicy(newLogMergePolicy());
429429
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
430430
Document doc = new Document();
431-
doc.add(new SortedNumericDocValuesField("sndv", 1));
431+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
432432
iw.addDocument(doc);
433433
doc = new Document();
434-
doc.add(new SortedNumericDocValuesField("sndv", 1));
434+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
435435
iw.addDocument(doc);
436436
iw.forceMerge(1);
437437
doc = new Document();
438-
doc.add(new SortedNumericDocValuesField("sndv", 2));
438+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
439439
iw.addDocument(doc);
440440
doc = new Document();
441-
doc.add(new SortedNumericDocValuesField("sndv", 2));
441+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
442442
iw.addDocument(doc);
443443
iw.forceMerge(1);
444444
iw.close();
@@ -487,18 +487,18 @@ public void testStarKeywordDocValuesWithMissingDocsInMixedSegments() throws IOEx
487487
conf.setMergePolicy(newLogMergePolicy());
488488
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf);
489489
Document doc = new Document();
490-
doc.add(new SortedNumericDocValuesField("sndv", 1));
490+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
491491
iw.addDocument(doc);
492492
doc = new Document();
493-
doc.add(new SortedNumericDocValuesField("sndv", 1));
493+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 1));
494494
iw.addDocument(doc);
495495
iw.forceMerge(1);
496496
doc = new Document();
497-
doc.add(new SortedNumericDocValuesField("sndv", 2));
497+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
498498
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
499499
iw.addDocument(doc);
500500
doc = new Document();
501-
doc.add(new SortedNumericDocValuesField("sndv", 2));
501+
doc.add(SortedNumericDocValuesField.indexedField("sndv", 2));
502502
iw.addDocument(doc);
503503
iw.forceMerge(1);
504504
iw.close();

0 commit comments

Comments
 (0)