Skip to content

Commit 7b50618

Browse files
romseygeekromseygeek
authored
Allow empty null values for date and IP field mappers (#62487)
In #57666 we changed when null_value was parsed for ip and date fields. Previously, the null value was stored as a string, and parsed into a date or InetAddress whenever a document containing a null value was encountered. Now, the values are parsed when the mappings are built, which means that bad values are detected up front; if you try and add a mapping with a badly-parsed ip or date for a null_value, the mapping will be rejected. This causes problems for upgrades in the case when you have a badly-formed null_value in a pre-7.9 cluster. This commit fixes the upgrade case by changing the logic to only fail on indexes created in 8x and later. For earlier indexes, we log a warning on the badly formed value and ignore it, replicating the earlier behaviour. Fixes #62363
1 parent 956d78a commit 7b50618

File tree

7 files changed

+140
-195
lines changed

7 files changed

+140
-195
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,10 @@
3232
import org.apache.lucene.search.Query;
3333
import org.apache.lucene.search.TermQuery;
3434
import org.elasticsearch.ElasticsearchParseException;
35+
import org.elasticsearch.Version;
3536
import org.elasticsearch.common.Nullable;
3637
import org.elasticsearch.common.geo.ShapeRelation;
38+
import org.elasticsearch.common.logging.DeprecationLogger;
3739
import org.elasticsearch.common.lucene.BytesRefs;
3840
import org.elasticsearch.common.time.DateFormatter;
3941
import org.elasticsearch.common.time.DateFormatters;
@@ -70,6 +72,8 @@
7072
/** A {@link FieldMapper} for dates. */
7173
public final class DateFieldMapper extends ParametrizedFieldMapper {
7274

75+
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);
76+
7377
public static final String CONTENT_TYPE = "date";
7478
public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
7579
public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis");
@@ -200,10 +204,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
200204
private final Parameter<Boolean> ignoreMalformed;
201205

202206
private final Resolution resolution;
207+
private final Version indexCreatedVersion;
203208

204-
public Builder(String name, Resolution resolution, DateFormatter dateFormatter, boolean ignoreMalformedByDefault) {
209+
public Builder(String name, Resolution resolution, DateFormatter dateFormatter,
210+
boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
205211
super(name);
206212
this.resolution = resolution;
213+
this.indexCreatedVersion = indexCreatedVersion;
207214
this.ignoreMalformed
208215
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
209216
if (dateFormatter != null) {
@@ -231,9 +238,14 @@ private Long parseNullValue(DateFieldType fieldType) {
231238
}
232239
try {
233240
return fieldType.parse(nullValue.getValue());
234-
}
235-
catch (Exception e) {
236-
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
241+
} catch (Exception e) {
242+
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
243+
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
244+
} else {
245+
DEPRECATION_LOGGER.deprecate("date_mapper_null_field", "Error parsing [" + nullValue.getValue()
246+
+ "] as date in [null_value] on field [" + name() + "]); [null_value] will be ignored");
247+
return null;
248+
}
237249
}
238250
}
239251

@@ -250,12 +262,12 @@ public DateFieldMapper build(BuilderContext context) {
250262

251263
public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
252264
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
253-
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault);
265+
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
254266
});
255267

256268
public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
257269
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
258-
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault);
270+
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
259271
});
260272

261273
public static final class DateFieldType extends MappedFieldType {
@@ -517,6 +529,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
517529
private final Resolution resolution;
518530

519531
private final boolean ignoreMalformedByDefault;
532+
private final Version indexCreatedVersion;
520533

521534
private DateFieldMapper(
522535
String simpleName,
@@ -537,11 +550,12 @@ private DateFieldMapper(
537550
this.nullValue = nullValue;
538551
this.resolution = resolution;
539552
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue();
553+
this.indexCreatedVersion = builder.indexCreatedVersion;
540554
}
541555

542556
@Override
543557
public ParametrizedFieldMapper.Builder getMergeBuilder() {
544-
return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault).init(this);
558+
return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault, indexCreatedVersion).init(this);
545559
}
546560

547561
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.lucene.document.Field;
2323
import org.apache.lucene.index.IndexableField;
2424
import org.elasticsearch.ElasticsearchParseException;
25+
import org.elasticsearch.Version;
2526
import org.elasticsearch.common.Strings;
2627
import org.elasticsearch.common.collect.Tuple;
2728
import org.elasticsearch.common.settings.Settings;
@@ -680,7 +681,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(final ParseContex
680681
if (builder == null) {
681682
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
682683
builder = new DateFieldMapper.Builder(currentFieldName, DateFieldMapper.Resolution.MILLISECONDS,
683-
dateTimeFormatter, ignoreMalformed);
684+
dateTimeFormatter, ignoreMalformed, Version.indexCreated(context.indexSettings().getSettings()));
684685
}
685686
return builder;
686687

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,11 @@
3030
import org.apache.lucene.search.TermQuery;
3131
import org.apache.lucene.util.ArrayUtil;
3232
import org.apache.lucene.util.BytesRef;
33+
import org.elasticsearch.Version;
3334
import org.elasticsearch.common.Nullable;
3435
import org.elasticsearch.common.collect.Tuple;
36+
import org.elasticsearch.common.logging.DeprecationLogger;
3537
import org.elasticsearch.common.network.InetAddresses;
36-
import org.elasticsearch.common.network.NetworkAddress;
3738
import org.elasticsearch.index.fielddata.IndexFieldData;
3839
import org.elasticsearch.index.fielddata.ScriptDocValues;
3940
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
@@ -55,6 +56,8 @@
5556
/** A {@link FieldMapper} for ip addresses. */
5657
public class IpFieldMapper extends ParametrizedFieldMapper {
5758

59+
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IpFieldMapper.class);
60+
5861
public static final String CONTENT_TYPE = "ip";
5962

6063
private static IpFieldMapper toType(FieldMapper in) {
@@ -68,36 +71,48 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
6871
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
6972

7073
private final Parameter<Boolean> ignoreMalformed;
71-
private final Parameter<InetAddress> nullValue = new Parameter<>("null_value", false, () -> null,
72-
(n, c, o) -> o == null ? null : InetAddresses.forString(o.toString()), m -> toType(m).nullValue)
73-
.setSerializer((b, f, v) -> {
74-
if (v == null) {
75-
b.nullField(f);
76-
} else {
77-
b.field(f, InetAddresses.toAddrString(v));
78-
}
79-
}, NetworkAddress::format)
80-
.acceptsNull();
74+
private final Parameter<String> nullValue
75+
= Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull();
8176

8277
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
8378

8479
private final boolean ignoreMalformedByDefault;
80+
private final Version indexCreatedVersion;
8581

86-
public Builder(String name, boolean ignoreMalformedByDefault) {
82+
public Builder(String name, boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
8783
super(name);
8884
this.ignoreMalformedByDefault = ignoreMalformedByDefault;
85+
this.indexCreatedVersion = indexCreatedVersion;
8986
this.ignoreMalformed
9087
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
9188
}
9289

93-
Builder nullValue(InetAddress nullValue) {
90+
Builder nullValue(String nullValue) {
9491
this.nullValue.setValue(nullValue);
9592
return this;
9693
}
9794

95+
private InetAddress parseNullValue() {
96+
String nullValueAsString = nullValue.getValue();
97+
if (nullValueAsString == null) {
98+
return null;
99+
}
100+
try {
101+
return InetAddresses.forString(nullValueAsString);
102+
} catch (Exception e) {
103+
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
104+
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
105+
} else {
106+
DEPRECATION_LOGGER.deprecate("ip_mapper_null_field", "Error parsing [" + nullValue.getValue()
107+
+ "] as IP in [null_value] on field [" + name() + "]); [null_value] will be ignored");
108+
return null;
109+
}
110+
}
111+
}
112+
98113
@Override
99114
protected List<Parameter<?>> getParameters() {
100-
return List.of(indexed, hasDocValues, stored, ignoreMalformed, nullValue);
115+
return List.of(indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta);
101116
}
102117

103118
@Override
@@ -111,7 +126,7 @@ public IpFieldMapper build(BuilderContext context) {
111126

112127
public static final TypeParser PARSER = new TypeParser((n, c) -> {
113128
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
114-
return new Builder(n, ignoreMalformedByDefault);
129+
return new Builder(n, ignoreMalformedByDefault, c.indexVersionCreated());
115130
});
116131

117132
public static final class IpFieldType extends SimpleMappedFieldType {
@@ -322,9 +337,12 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
322337
private final boolean hasDocValues;
323338
private final boolean stored;
324339
private final boolean ignoreMalformed;
340+
325341
private final InetAddress nullValue;
342+
private final String nullValueAsString;
326343

327344
private final boolean ignoreMalformedByDefault;
345+
private final Version indexCreatedVersion;
328346

329347
private IpFieldMapper(
330348
String simpleName,
@@ -338,7 +356,9 @@ private IpFieldMapper(
338356
this.hasDocValues = builder.hasDocValues.getValue();
339357
this.stored = builder.stored.getValue();
340358
this.ignoreMalformed = builder.ignoreMalformed.getValue();
341-
this.nullValue = builder.nullValue.getValue();
359+
this.nullValue = builder.parseNullValue();
360+
this.nullValueAsString = builder.nullValue.getValue();
361+
this.indexCreatedVersion = builder.indexCreatedVersion;
342362
}
343363

344364
@Override
@@ -424,6 +444,6 @@ protected Object parseSourceValue(Object value) {
424444

425445
@Override
426446
public ParametrizedFieldMapper.Builder getMergeBuilder() {
427-
return new Builder(simpleName(), ignoreMalformedByDefault).init(this);
447+
return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this);
428448
}
429449
}

‎server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
553553
try {
554554
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
555555
DateFieldMapper dateFieldMapper
556-
= new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false).build(builderContext);
556+
= new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT)
557+
.build(builderContext);
557558
MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class);
558559
when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp");
559560
MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class);

‎server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,18 @@ public void testNanosNullValue() throws IOException {
220220
assertFalse(dvField.fieldType().stored());
221221
}
222222

223-
public void testBadNullValue() {
223+
public void testBadNullValue() throws IOException {
224224

225225
MapperParsingException e = expectThrows(MapperParsingException.class,
226-
() -> createDocumentMapper(fieldMapping(b -> b.field("type", "date").field("null_value", ""))));
226+
() -> createDocumentMapper(Version.V_8_0_0, fieldMapping(b -> b.field("type", "date").field("null_value", "foo"))));
227227

228228
assertThat(e.getMessage(),
229-
equalTo("Failed to parse mapping: Error parsing [null_value] on field [field]: cannot parse empty date"));
229+
equalTo("Failed to parse mapping: Error parsing [null_value] on field [field]: " +
230+
"failed to parse date field [foo] with format [strict_date_optional_time||epoch_millis]"));
231+
232+
createDocumentMapper(Version.V_7_9_0, fieldMapping(b -> b.field("type", "date").field("null_value", "foo")));
233+
234+
assertWarnings("Error parsing [foo] as date in [null_value] on field [field]); [null_value] will be ignored");
230235
}
231236

232237
public void testNullConfigValuesFail() {
@@ -364,7 +369,8 @@ private DateFieldMapper createMapper(Resolution resolution, String format, Strin
364369
mapping.put("null_value", nullValue);
365370
}
366371

367-
DateFieldMapper.Builder builder = new DateFieldMapper.Builder("field", resolution, null, false);
372+
DateFieldMapper.Builder builder
373+
= new DateFieldMapper.Builder("field", resolution, null, false, Version.CURRENT);
368374
builder.parse("field", null, mapping);
369375
return builder.build(context);
370376
}

0 commit comments

Comments
 (0)