Skip to content

Commit cf27239

Browse files
asimmahmood1Aditya Khera
asimmahmood1
authored and
Aditya Khera
committed
Add skip_list parameter to Numeric Field Mappers (default false) (opensearch-project#18889)
* Add skip_list parameter to Numeric Field Mappers - Add skip_list parameter to numeric field mappers (default false) - Add mapping unit tests for skip_list functionality Signed-off-by: Asim Mahmood <asim.seng@gmail.com> * Fix test Signed-off-by: Asim Mahmood <asim.seng@gmail.com> * Remove unused code Signed-off-by: Asim Mahmood <asim.seng@gmail.com> --------- Signed-off-by: Asim Mahmood <asim.seng@gmail.com> Signed-off-by: Asim M <asim.seng@gmail.com>
1 parent acf51f9 commit cf27239

File tree

12 files changed

+168
-39
lines changed

12 files changed

+168
-39
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5454
- [Derived Source] Add integration of derived source feature across various paths like get/search/recovery ([#18565](https://github.com/opensearch-project/OpenSearch/pull/18565))
5555
- Supporting Scripted Metric Aggregation when reducing aggregations in InternalValueCount and InternalAvg ([18411](https://github.com/opensearch-project/OpenSearch/pull18411)))
5656
- Support `search_after` numeric queries with Approximation Framework ([#18896](https://github.com/opensearch-project/OpenSearch/pull/18896))
57+
- Add skip_list parameter to Numeric Field Mappers (default false) ([#18889](https://github.com/opensearch-project/OpenSearch/pull/18889))
5758

5859
### Changed
5960
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -484,7 +484,14 @@ protected void parseCreateField(ParseContext context) throws IOException {
484484
}
485485
long scaledValue = Math.round(doubleValue * scalingFactor);
486486

487-
List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), scaledValue, indexed, hasDocValues, stored);
487+
List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
488+
fieldType().name(),
489+
scaledValue,
490+
indexed,
491+
hasDocValues,
492+
false,
493+
stored
494+
);
488495
context.doc().addAll(fields);
489496

490497
if (hasDocValues == false && (indexed || stored)) {

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ protected void parseCreateField(ParseContext context) throws IOException {
173173
tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
174174
}
175175

176-
context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, store));
176+
context.doc()
177+
.addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, false, store));
177178
}
178179

179180
/**

‎modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ public void testDuel() throws Exception {
321321
document.add(new TextField(entry.getKey(), value, Field.Store.NO));
322322
}
323323
for (Integer intValue : intValues) {
324-
List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false);
324+
List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false, false);
325325
for (Field numberField : numberFields) {
326326
document.add(numberField);
327327
}
@@ -449,6 +449,7 @@ public void testDuel2() throws Exception {
449449
between(range[0], range[1]),
450450
true,
451451
true,
452+
false,
452453
false
453454
);
454455
for (Field numberField : numberFields) {

‎plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public void postParse(ParseContext context) throws IOException {
9999
return;
100100
}
101101
final int value = context.sourceToParse().source().length();
102-
context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, true));
102+
context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));
103103
}
104104

105105
@Override

‎server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 118 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import org.opensearch.common.settings.Setting;
5959
import org.opensearch.common.settings.Setting.Property;
6060
import org.opensearch.common.settings.Settings;
61+
import org.opensearch.common.xcontent.support.XContentMapValues;
6162
import org.opensearch.core.common.bytes.BytesArray;
6263
import org.opensearch.core.xcontent.XContentBuilder;
6364
import org.opensearch.core.xcontent.XContentParser;
@@ -118,6 +119,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
118119
private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
119120
private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
120121
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
122+
private final Parameter<Boolean> skiplist = new Parameter<>(
123+
"skip_list",
124+
false,
125+
() -> false,
126+
(n, c, o) -> XContentMapValues.nodeBooleanValue(o),
127+
m -> toType(m).skiplist
128+
);
121129

122130
private final Parameter<Explicit<Boolean>> ignoreMalformed;
123131
private final Parameter<Explicit<Boolean>> coerce;
@@ -169,7 +177,7 @@ public Builder docValues(boolean hasDocValues) {
169177

170178
@Override
171179
protected List<Parameter<?>> getParameters() {
172-
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, meta);
180+
return Arrays.asList(indexed, hasDocValues, stored, skiplist, ignoreMalformed, coerce, nullValue, meta);
173181
}
174182

175183
@Override
@@ -416,13 +424,26 @@ public Query rangeQuery(
416424
}
417425

418426
@Override
419-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
427+
public List<Field> createFields(
428+
String name,
429+
Number value,
430+
boolean indexed,
431+
boolean docValued,
432+
boolean skiplist,
433+
boolean stored
434+
) {
420435
List<Field> fields = new ArrayList<>();
421436
if (indexed) {
422437
fields.add(new HalfFloatPoint(name, value.floatValue()));
423438
}
424439
if (docValued) {
425-
fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
440+
if (skiplist) {
441+
fields.add(
442+
SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue()))
443+
);
444+
} else {
445+
fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
446+
}
426447
}
427448
if (stored) {
428449
fields.add(new StoredField(name, value.floatValue()));
@@ -603,13 +624,24 @@ public Query rangeQuery(
603624
}
604625

605626
@Override
606-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
627+
public List<Field> createFields(
628+
String name,
629+
Number value,
630+
boolean indexed,
631+
boolean docValued,
632+
boolean skiplist,
633+
boolean stored
634+
) {
607635
List<Field> fields = new ArrayList<>();
608636
if (indexed) {
609637
fields.add(new FloatPoint(name, value.floatValue()));
610638
}
611639
if (docValued) {
612-
fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
640+
if (skiplist) {
641+
fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(value.floatValue())));
642+
} else {
643+
fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
644+
}
613645
}
614646
if (stored) {
615647
fields.add(new StoredField(name, value.floatValue()));
@@ -766,13 +798,24 @@ public Query rangeQuery(
766798
}
767799

768800
@Override
769-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
801+
public List<Field> createFields(
802+
String name,
803+
Number value,
804+
boolean indexed,
805+
boolean docValued,
806+
boolean skiplist,
807+
boolean stored
808+
) {
770809
List<Field> fields = new ArrayList<>();
771810
if (indexed) {
772811
fields.add(new DoublePoint(name, value.doubleValue()));
773812
}
774813
if (docValued) {
775-
fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
814+
if (skiplist) {
815+
fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
816+
} else {
817+
fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
818+
}
776819
}
777820
if (stored) {
778821
fields.add(new StoredField(name, value.doubleValue()));
@@ -878,8 +921,15 @@ public Query rangeQuery(
878921
}
879922

880923
@Override
881-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
882-
return INTEGER.createFields(name, value, indexed, docValued, stored);
924+
public List<Field> createFields(
925+
String name,
926+
Number value,
927+
boolean indexed,
928+
boolean docValued,
929+
boolean skiplist,
930+
boolean stored
931+
) {
932+
return INTEGER.createFields(name, value, indexed, docValued, skiplist, stored);
883933
}
884934

885935
@Override
@@ -974,8 +1024,15 @@ public Query rangeQuery(
9741024
}
9751025

9761026
@Override
977-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
978-
return INTEGER.createFields(name, value, indexed, docValued, stored);
1027+
public List<Field> createFields(
1028+
String name,
1029+
Number value,
1030+
boolean indexed,
1031+
boolean docValued,
1032+
boolean skiplist,
1033+
boolean stored
1034+
) {
1035+
return INTEGER.createFields(name, value, indexed, docValued, skiplist, stored);
9791036
}
9801037

9811038
@Override
@@ -1185,13 +1242,24 @@ public Query rangeQuery(
11851242
}
11861243

11871244
@Override
1188-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
1245+
public List<Field> createFields(
1246+
String name,
1247+
Number value,
1248+
boolean indexed,
1249+
boolean docValued,
1250+
boolean skiplist,
1251+
boolean stored
1252+
) {
11891253
List<Field> fields = new ArrayList<>();
11901254
if (indexed) {
11911255
fields.add(new IntPoint(name, value.intValue()));
11921256
}
11931257
if (docValued) {
1194-
fields.add(new SortedNumericDocValuesField(name, value.intValue()));
1258+
if (skiplist) {
1259+
fields.add(SortedNumericDocValuesField.indexedField(name, value.intValue()));
1260+
} else {
1261+
fields.add(new SortedNumericDocValuesField(name, value.intValue()));
1262+
}
11951263
}
11961264
if (stored) {
11971265
fields.add(new StoredField(name, value.intValue()));
@@ -1344,13 +1412,24 @@ public Query rangeQuery(
13441412
}
13451413

13461414
@Override
1347-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
1415+
public List<Field> createFields(
1416+
String name,
1417+
Number value,
1418+
boolean indexed,
1419+
boolean docValued,
1420+
boolean skiplist,
1421+
boolean stored
1422+
) {
13481423
List<Field> fields = new ArrayList<>();
13491424
if (indexed) {
13501425
fields.add(new LongPoint(name, value.longValue()));
13511426
}
13521427
if (docValued) {
1353-
fields.add(new SortedNumericDocValuesField(name, value.longValue()));
1428+
if (skiplist) {
1429+
fields.add(SortedNumericDocValuesField.indexedField(name, value.longValue()));
1430+
} else {
1431+
fields.add(new SortedNumericDocValuesField(name, value.longValue()));
1432+
}
13541433
}
13551434
if (stored) {
13561435
fields.add(new StoredField(name, value.longValue()));
@@ -1491,7 +1570,14 @@ public Query rangeQuery(
14911570
}
14921571

14931572
@Override
1494-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
1573+
public List<Field> createFields(
1574+
String name,
1575+
Number value,
1576+
boolean indexed,
1577+
boolean docValued,
1578+
boolean skiplist,
1579+
boolean stored
1580+
) {
14951581
List<Field> fields = new ArrayList<>();
14961582
final BigInteger v = Numbers.toUnsignedLongExact(value);
14971583

@@ -1500,7 +1586,11 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
15001586
}
15011587

15021588
if (docValued) {
1503-
fields.add(new SortedNumericDocValuesField(name, v.longValue()));
1589+
if (skiplist) {
1590+
fields.add(SortedNumericDocValuesField.indexedField(name, v.longValue()));
1591+
} else {
1592+
fields.add(new SortedNumericDocValuesField(name, v.longValue()));
1593+
}
15041594
}
15051595

15061596
if (stored) {
@@ -1569,7 +1659,14 @@ public abstract Query rangeQuery(
15691659

15701660
public abstract Number parsePoint(byte[] value);
15711661

1572-
public abstract List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored);
1662+
public abstract List<Field> createFields(
1663+
String name,
1664+
Number value,
1665+
boolean indexed,
1666+
boolean docValued,
1667+
boolean skiplist,
1668+
boolean stored
1669+
);
15731670

15741671
abstract Number valueForSearch(String value);
15751672

@@ -1987,6 +2084,7 @@ public Number parse(Object value) {
19872084
private final boolean indexed;
19882085
private final boolean hasDocValues;
19892086
private final boolean stored;
2087+
private final boolean skiplist;
19902088
private final Explicit<Boolean> ignoreMalformed;
19912089
private final Explicit<Boolean> coerce;
19922090
private final Number nullValue;
@@ -2000,6 +2098,7 @@ private NumberFieldMapper(String simpleName, MappedFieldType mappedFieldType, Mu
20002098
this.indexed = builder.indexed.getValue();
20012099
this.hasDocValues = builder.hasDocValues.getValue();
20022100
this.stored = builder.stored.getValue();
2101+
this.skiplist = builder.skiplist.getValue();
20032102
this.ignoreMalformed = builder.ignoreMalformed.getValue();
20042103
this.coerce = builder.coerce.getValue();
20052104
this.nullValue = builder.nullValue.getValue();
@@ -2068,7 +2167,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
20682167
numericValue = fieldType().type.parse(value, coerce.value());
20692168
}
20702169

2071-
context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, stored));
2170+
context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, skiplist, stored));
20722171

20732172
if (hasDocValues == false && (stored || indexed)) {
20742173
createFieldNamesField(context);

‎server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public void testSingleValued() throws IOException {
5555
// we need the default codec to check for singletons
5656
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec()));
5757
Document doc = new Document();
58-
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
58+
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false, false)) {
5959
doc.add(f);
6060
}
6161
w.addDocument(doc);
@@ -74,10 +74,10 @@ public void testMultiValued() throws IOException {
7474
Directory dir = newDirectory();
7575
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
7676
Document doc = new Document();
77-
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
77+
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false, false)) {
7878
doc.add(f);
7979
}
80-
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false)) {
80+
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false, false)) {
8181
doc.add(f);
8282
}
8383
w.addDocument(doc);

0 commit comments

Comments
 (0)