Skip to content

Commit 6e2ff5c

Browse files
mishailandrross
mishail
and
andrross
authored
[fixes #16599] Field-level ignore_malformed should override index-level setting (#18706)
* [fixes #16599] Field-level `ignore_malformed` should override index-level setting Signed-off-by: Mikhail Stepura <mstepura@apple.com> * CHANGELOG.md Signed-off-by: Mikhail Stepura <mstepura@apple.com> * Update CHANGELOG.md Co-authored-by: Andrew Ross <andrross@amazon.com> Signed-off-by: Mikhail Stepura <mikhail.stepura@outlook.com> * API level test Signed-off-by: Mikhail Stepura <mstepura@apple.com> * Ignore for the versions without the fix Signed-off-by: Mikhail Stepura <mstepura@apple.com> * Cleanup Signed-off-by: Mikhail Stepura <mstepura@apple.com> --------- Signed-off-by: Mikhail Stepura <mstepura@apple.com> Signed-off-by: Andrew Ross <andrross@amazon.com> Signed-off-by: Mikhail Stepura <mikhail.stepura@outlook.com> Co-authored-by: Andrew Ross <andrross@amazon.com>
1 parent a599cd4 commit 6e2ff5c

File tree

14 files changed

+136
-55
lines changed

14 files changed

+136
-55
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
7575
- Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640))
7676
- Fix Replication lag computation ([#18602](https://github.com/opensearch-project/OpenSearch/pull/18602))
7777
- Fix max_score is null when sorting on score firstly ([#18715](https://github.com/opensearch-project/OpenSearch/pull/18715))
78+
- Field-level ignore_malformed should override index-level setting ([#18706](https://github.com/opensearch-project/OpenSearch/pull/18706))
7879
- Fixed Staggered merge - load average replace with AverageTrackers, some Default thresholds modified ([#18666](https://github.com/opensearch-project/OpenSearch/pull/18666))
7980
- Use `new SecureRandom()` to avoid blocking ([18729](https://github.com/opensearch-project/OpenSearch/issues/18729))
8081

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,8 +395,9 @@ boolean coerce() {
395395
return coerce.value();
396396
}
397397

398-
boolean ignoreMalformed() {
399-
return ignoreMalformed.value();
398+
@Override
399+
protected Explicit<Boolean> ignoreMalformed() {
400+
return ignoreMalformed;
400401
}
401402

402403
@Override

‎modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
9090
checker.registerUpdateCheck(b -> b.field("coerce", false), m -> assertFalse(((ScaledFloatFieldMapper) m).coerce()));
9191
checker.registerUpdateCheck(
9292
b -> b.field("ignore_malformed", true),
93-
m -> assertTrue(((ScaledFloatFieldMapper) m).ignoreMalformed())
93+
m -> assertTrue(((ScaledFloatFieldMapper) m).ignoreMalformed().value())
9494
);
9595
}
9696

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
setup:
33
- do:
44
indices.create:
5-
index: test
5+
index: test
66
body:
77
mappings:
88
properties:
@@ -13,6 +13,19 @@ setup:
1313
my_ip:
1414
type: ip
1515
ignore_malformed: true
16+
- do:
17+
indices.create:
18+
index: index_level_setting
19+
body:
20+
settings:
21+
index.mapping.ignore_malformed: true
22+
mappings:
23+
properties:
24+
my_date:
25+
type: date
26+
ignore_malformed: false
27+
my_ip:
28+
type: ip
1629

1730
- do:
1831
index:
@@ -40,6 +53,7 @@ setup:
4053

4154
- do:
4255
search:
56+
index: test
4357
rest_total_hits_as_int: true
4458
body: { query: { exists: { "field": "_ignored" } } }
4559

@@ -50,6 +64,7 @@ setup:
5064

5165
- do:
5266
search:
67+
index: test
5368
rest_total_hits_as_int: true
5469
body: { query: { term: { "_ignored": "my_date" } } }
5570

@@ -60,6 +75,7 @@ setup:
6075

6176
- do:
6277
search:
78+
index: test
6379
rest_total_hits_as_int: true
6480
body: { query: { terms: { "_ignored": [ "my_date", "my_ip" ] } } }
6581

@@ -70,6 +86,7 @@ setup:
7086

7187
- do:
7288
search:
89+
index: test
7390
rest_total_hits_as_int: true
7491
body: { query: { ids: { "values": [ "3" ] } } }
7592

@@ -81,9 +98,21 @@ setup:
8198

8299
- do:
83100
search:
101+
index: test
84102
rest_total_hits_as_int: true
85103
stored_fields: [ "my_date" ]
86104
body: { query: { ids: { "values": [ "3" ] } } }
87105

88106
- length: { hits.hits: 1 }
89107
- is_true: hits.hits.0._ignored
108+
109+
---
110+
"field-level setting shall override index-level setting":
111+
- skip:
112+
version: " - 3.1.99"
113+
reason: "Versions pre-3.2.0 are affected by the bug"
114+
- do:
115+
catch: /failed to parse field \[my_date\] of type \[date\]/
116+
index:
117+
index: index_level_setting
118+
body: { "my_date": "bar", "my_ip": "quux" }

‎server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,8 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
452452
}
453453
}
454454

455-
public Explicit<Boolean> ignoreMalformed() {
455+
@Override
456+
protected Explicit<Boolean> ignoreMalformed() {
456457
return ignoreMalformed;
457458
}
458459

‎server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.lucene.search.Query;
4545
import org.opensearch.OpenSearchParseException;
4646
import org.opensearch.Version;
47+
import org.opensearch.common.Explicit;
4748
import org.opensearch.common.Nullable;
4849
import org.opensearch.common.geo.ShapeRelation;
4950
import org.opensearch.common.logging.DeprecationLogger;
@@ -294,7 +295,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
294295

295296
private final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null)
296297
.acceptsNull();
297-
private final Parameter<Boolean> ignoreMalformed;
298+
private final Parameter<Explicit<Boolean>> ignoreMalformed;
298299

299300
private final Resolution resolution;
300301
private final Version indexCreatedVersion;
@@ -309,7 +310,12 @@ public Builder(
309310
super(name);
310311
this.resolution = resolution;
311312
this.indexCreatedVersion = indexCreatedVersion;
312-
this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
313+
this.ignoreMalformed = Parameter.explicitBoolParam(
314+
"ignore_malformed",
315+
true,
316+
m -> toType(m).ignoreMalformed,
317+
ignoreMalformedByDefault
318+
);
313319
if (dateFormatter != null) {
314320
this.format.setValue(dateFormatter.pattern());
315321
this.printFormat.setValue(dateFormatter.printPattern());
@@ -721,7 +727,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
721727
private final Locale locale;
722728
private final String format;
723729
private final String printFormat;
724-
private final boolean ignoreMalformed;
730+
private final Explicit<Boolean> ignoreMalformed;
725731
private final Long nullValue;
726732
private final String nullValueAsString;
727733
private final Resolution resolution;
@@ -749,7 +755,7 @@ private DateFieldMapper(
749755
this.nullValueAsString = builder.nullValue.getValue();
750756
this.nullValue = nullValue;
751757
this.resolution = resolution;
752-
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue();
758+
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
753759
this.indexCreatedVersion = builder.indexCreatedVersion;
754760
}
755761

@@ -797,7 +803,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
797803
try {
798804
timestamp = fieldType().parse(dateAsString);
799805
} catch (IllegalArgumentException | OpenSearchParseException | DateTimeException | ArithmeticException e) {
800-
if (ignoreMalformed) {
806+
if (ignoreMalformed().value()) {
801807
context.addIgnoredField(mappedFieldType.name());
802808
return;
803809
} else {
@@ -819,11 +825,12 @@ protected void parseCreateField(ParseContext context) throws IOException {
819825
}
820826
}
821827

822-
public boolean getIgnoreMalformed() {
823-
return ignoreMalformed;
824-
}
825-
826828
public Long getNullValue() {
827829
return nullValue;
828830
}
831+
832+
@Override
833+
protected Explicit<Boolean> ignoreMalformed() {
834+
return ignoreMalformed;
835+
}
829836
}

‎server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.opensearch.index.mapper;
1010

1111
import org.apache.lucene.index.IndexableField;
12+
import org.opensearch.common.Explicit;
1213
import org.opensearch.common.time.DateFormatter;
1314
import org.opensearch.core.xcontent.XContentBuilder;
1415
import org.opensearch.index.analysis.IndexAnalyzers;
@@ -71,7 +72,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
7172
m -> toType(m).format,
7273
getDefaultDateTimeFormatter().pattern()
7374
);
74-
private final Parameter<Boolean> ignoreMalformed;
75+
private final Parameter<Explicit<Boolean>> ignoreMalformed;
7576

7677
private static Map<String, Object> parseProperties(String name, Object propertiesObject) {
7778
if (propertiesObject instanceof Map == false) {
@@ -114,7 +115,12 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, DateFormatter default
114115
if (defaultDateFormatter != null) {
115116
this.format.setValue(defaultDateFormatter.pattern());
116117
}
117-
this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, defaultIgnoreMalformed);
118+
this.ignoreMalformed = Parameter.explicitBoolParam(
119+
"ignore_malformed",
120+
true,
121+
m -> toType(m).ignoreMalformed,
122+
defaultIgnoreMalformed
123+
);
118124
}
119125

120126
public Builder(
@@ -136,7 +142,7 @@ public Builder(
136142
this.format.setValue(derivedField.getFormat());
137143
}
138144
if (derivedField.getIgnoreMalformed()) {
139-
this.ignoreMalformed.setValue(derivedField.getIgnoreMalformed());
145+
this.ignoreMalformed.setValue(new Explicit<>(derivedField.getIgnoreMalformed(), true));
140146
}
141147
}
142148

@@ -158,7 +164,7 @@ public DerivedFieldMapper build(BuilderContext context) {
158164
derivedField.setFormat(format.getValue());
159165
}
160166
if (ignoreMalformed.isConfigured()) {
161-
derivedField.setIgnoreMalformed(ignoreMalformed.getValue());
167+
derivedField.setIgnoreMalformed(ignoreMalformed.getValue().value());
162168
}
163169
FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context, indexAnalyzers);
164170
Function<Object, IndexableField> fieldFunction = DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(
@@ -193,7 +199,7 @@ public DerivedFieldMapper build(BuilderContext context) {
193199
private final Script script;
194200
private final String prefilterField;
195201
private final Map<String, Object> properties;
196-
private final boolean ignoreMalformed;
202+
private final Explicit<Boolean> ignoreMalformed;
197203
private final boolean defaultIgnoreMalformed;
198204
private final DateFormatter defaultDateFormatter;
199205
private final String format;
@@ -257,4 +263,8 @@ public Script getScript() {
257263
return script;
258264
}
259265

266+
@Override
267+
protected Explicit<Boolean> ignoreMalformed() {
268+
return ignoreMalformed;
269+
}
260270
}

‎server/src/main/java/org/opensearch/index/mapper/FieldMapper.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,15 @@
3636
import org.apache.lucene.document.FieldType;
3737
import org.apache.lucene.index.IndexOptions;
3838
import org.apache.lucene.index.LeafReader;
39+
import org.opensearch.common.Explicit;
3940
import org.opensearch.common.annotation.PublicApi;
4041
import org.opensearch.common.settings.Setting;
4142
import org.opensearch.common.settings.Setting.Property;
4243
import org.opensearch.common.settings.Settings;
4344
import org.opensearch.core.xcontent.AbstractXContentParser;
4445
import org.opensearch.core.xcontent.XContentBuilder;
4546
import org.opensearch.core.xcontent.XContentParser;
47+
import org.opensearch.index.IndexSettings;
4648
import org.opensearch.index.analysis.NamedAnalyzer;
4749
import org.opensearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType;
4850

@@ -264,15 +266,28 @@ public boolean parsesArrayValue() {
264266
return false;
265267
}
266268

269+
private boolean shouldIgnoreMalformed(IndexSettings is) {
270+
if (ignoreMalformed() != null) {
271+
return ignoreMalformed().value();
272+
}
273+
274+
if (is == null) {
275+
return false;
276+
}
277+
278+
return IGNORE_MALFORMED_SETTING.get(is.getSettings());
279+
}
280+
267281
/**
268282
* Parse the field value using the provided {@link ParseContext}.
269283
*/
270284
public void parse(ParseContext context) throws IOException {
271285
try {
272286
parseCreateField(context);
273287
} catch (Exception e) {
274-
boolean ignore_malformed = false;
275-
if (context.indexSettings() != null) ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
288+
289+
boolean ignoreMalformed = shouldIgnoreMalformed(context.indexSettings());
290+
276291
String valuePreview = "";
277292
try {
278293
XContentParser parser = context.parser();
@@ -283,7 +298,7 @@ public void parse(ParseContext context) throws IOException {
283298
valuePreview = complexValue.toString();
284299
}
285300
} catch (Exception innerException) {
286-
if (ignore_malformed == false) {
301+
if (ignoreMalformed == false) {
287302
throw new MapperParsingException(
288303
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
289304
e,
@@ -294,7 +309,7 @@ public void parse(ParseContext context) throws IOException {
294309
}
295310
}
296311

297-
if (ignore_malformed == false) {
312+
if (ignoreMalformed == false) {
298313
throw new MapperParsingException(
299314
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
300315
e,
@@ -574,6 +589,13 @@ protected static String indexOptionToString(IndexOptions indexOption) {
574589

575590
protected abstract String contentType();
576591

592+
/**
593+
* @return null if a mapper doesn't support setting of `ignore_malformed`
594+
*/
595+
protected Explicit<Boolean> ignoreMalformed() {
596+
return null;
597+
}
598+
577599
/**
578600
* Method to create derived source generator for this field mapper, it is illegal to enable the
579601
* derived source feature and not implement this method for a field mapper

0 commit comments

Comments
 (0)