Skip to content

Commit fb10346

Browse files
dadoonetdadoonet
committed
[Mapper] Add ignore_missing option to timestamp
Related to elastic#9049. By default, the default value for `timestamp` is `now` which means the date the document was processed by the indexing chain. You can now reject documents which not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`): ```js { "tweet" : { "_timestamp" : { "enabled" : true, "ignore_missing" : false } } } ``` When you update the cluster to 1.5 or master, this index created with 1.4 we automatically migrate an index created with 1.4 to the 1.5 syntax. Let say you have defined this in elasticsearch 1.4.x: ```js DELETE test PUT test { "settings": { "number_of_shards": 1, "number_of_replicas": 0 } } PUT test/type/_mapping { "type" : { "_timestamp" : { "enabled" : true, "default" : null } } } ``` After migration, the mapping become: ```js { "test": { "mappings": { "type": { "_timestamp": { "enabled": true, "store": false, "ignore_missing": false }, "properties": {} } } } } ``` Closes elastic#8882.
1 parent fcb0186 commit fb10346

File tree

6 files changed

+210
-146
lines changed

6 files changed

+210
-146
lines changed

‎docs/reference/mapping/fields/timestamp-field.asciidoc‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,22 +91,20 @@ within the index request or in the `_source` document.
9191

9292
By default, the default value is `now` which means the date the document was processed by the indexing chain.
9393

94-
You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
94+
You can reject documents which do not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):
9595

9696
[source,js]
9797
--------------------------------------------------
9898
{
9999
"tweet" : {
100100
"_timestamp" : {
101101
"enabled" : true,
102-
"default" : null
102+
"ignore_missing" : false
103103
}
104104
}
105105
}
106106
--------------------------------------------------
107107

108-
If you don't provide any timestamp value, indexation will fail.
109-
110108
You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:
111109

112110
[source,js]
@@ -124,3 +122,10 @@ You can also set the default value to any date respecting <<mapping-timestamp-fi
124122

125123
If you don't provide any timestamp value, _timestamp will be set to this default value.
126124

125+
coming[1.5.0]
126+
127+
In elasticsearch 1.4, we allowed setting explicitly `"default":null` which is not possible anymore
128+
as we added a new `ignore_missing`setting.
129+
When reading an index created with elasticsearch 1.4 and using this, we automatically update it by
130+
removing `"default": null` and setting `"ignore_missing": false`
131+

‎src/main/java/org/elasticsearch/action/index/IndexRequest.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,11 @@
2020
package org.elasticsearch.action.index;
2121

2222
import com.google.common.base.Charsets;
23-
import org.elasticsearch.*;
24-
import org.elasticsearch.action.ActionRequest;
25-
import org.elasticsearch.action.ActionRequestValidationException;
26-
import org.elasticsearch.action.RoutingMissingException;
27-
import org.elasticsearch.action.TimestampParsingException;
28-
import org.elasticsearch.action.DocumentRequest;
23+
import org.elasticsearch.ElasticsearchException;
24+
import org.elasticsearch.ElasticsearchGenerationException;
25+
import org.elasticsearch.ElasticsearchIllegalArgumentException;
26+
import org.elasticsearch.ElasticsearchParseException;
27+
import org.elasticsearch.action.*;
2928
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
3029
import org.elasticsearch.client.Requests;
3130
import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -659,11 +658,13 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
659658
if (timestamp == null) {
660659
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
661660
if (mappingMd != null && mappingMd.timestamp() != null) {
661+
// If we explicitly ask to reject null timestamp
662+
if (mappingMd.timestamp().ignoreMissing() != null && mappingMd.timestamp().ignoreMissing() == false) {
663+
throw new TimestampParsingException("timestamp is required by mapping");
664+
}
662665
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
663666
}
664-
if (!Strings.hasText(defaultTimestamp)) {
665-
throw new TimestampParsingException("timestamp is required by mapping");
666-
}
667+
667668
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
668669
timestamp = Long.toString(System.currentTimeMillis());
669670
} else {

‎src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
177177

178178

179179
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
180-
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
180+
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null);
181181

182182
private final boolean enabled;
183183

@@ -191,7 +191,9 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
191191

192192
private final String defaultTimestamp;
193193

194-
public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
194+
private final Boolean ignoreMissing;
195+
196+
public Timestamp(boolean enabled, String path, String format, String defaultTimestamp, Boolean ignoreMissing) {
195197
this.enabled = enabled;
196198
this.path = path;
197199
if (path == null) {
@@ -202,6 +204,7 @@ public Timestamp(boolean enabled, String path, String format, String defaultTime
202204
this.format = format;
203205
this.dateTimeFormatter = Joda.forPattern(format);
204206
this.defaultTimestamp = defaultTimestamp;
207+
this.ignoreMissing = ignoreMissing;
205208
}
206209

207210
public boolean enabled() {
@@ -232,6 +235,10 @@ public boolean hasDefaultTimestamp() {
232235
return this.defaultTimestamp != null;
233236
}
234237

238+
public Boolean ignoreMissing() {
239+
return ignoreMissing;
240+
}
241+
235242
public FormatDateTimeFormatter dateTimeFormatter() {
236243
return this.dateTimeFormatter;
237244
}
@@ -247,6 +254,7 @@ public boolean equals(Object o) {
247254
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
248255
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
249256
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
257+
if (ignoreMissing != null ? !ignoreMissing.equals(timestamp.ignoreMissing) : timestamp.ignoreMissing != null) return false;
250258
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;
251259

252260
return true;
@@ -260,6 +268,7 @@ public int hashCode() {
260268
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
261269
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
262270
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
271+
result = 31 * result + (ignoreMissing != null ? ignoreMissing.hashCode() : 0);
263272
return result;
264273
}
265274
}
@@ -278,7 +287,9 @@ public MappingMetaData(DocumentMapper docMapper) {
278287
this.source = docMapper.mappingSource();
279288
this.id = new Id(docMapper.idFieldMapper().path());
280289
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
281-
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
290+
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(),
291+
docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp(),
292+
docMapper.timestampFieldMapper().ignoreMissing());
282293
this.hasParentField = docMapper.parentFieldMapper().active();
283294
}
284295

@@ -344,6 +355,7 @@ private void initMappers(Map<String, Object> withoutType) {
344355
String path = null;
345356
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
346357
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
358+
Boolean ignoreMissing = null;
347359
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
348360
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
349361
String fieldName = Strings.toUnderscoreCase(entry.getKey());
@@ -356,9 +368,11 @@ private void initMappers(Map<String, Object> withoutType) {
356368
format = fieldNode.toString();
357369
} else if (fieldName.equals("default") && fieldNode != null) {
358370
defaultTimestamp = fieldNode.toString();
371+
} else if (fieldName.equals("ignore_missing")) {
372+
ignoreMissing = nodeBooleanValue(fieldNode);
359373
}
360374
}
361-
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
375+
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
362376
} else {
363377
this.timestamp = Timestamp.EMPTY;
364378
}
@@ -542,6 +556,10 @@ public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws I
542556
out.writeOptionalString(mappingMd.timestamp().path());
543557
out.writeString(mappingMd.timestamp().format());
544558
out.writeOptionalString(mappingMd.timestamp().defaultTimestamp());
559+
// TODO Remove the test in elasticsearch 2.0.0
560+
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
561+
out.writeOptionalBoolean(mappingMd.timestamp().ignoreMissing());
562+
}
545563
out.writeBoolean(mappingMd.hasParentField());
546564
}
547565

@@ -579,7 +597,19 @@ public static MappingMetaData readFrom(StreamInput in) throws IOException {
579597
// routing
580598
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
581599
// timestamp
582-
final Timestamp timestamp = new Timestamp(in.readBoolean(), in.readOptionalString(), in.readString(), in.readOptionalString());
600+
601+
boolean enabled = in.readBoolean();
602+
String path = in.readOptionalString();
603+
String format = in.readString();
604+
String defaultTimestamp = in.readOptionalString();
605+
Boolean ignoreMissing = null;
606+
607+
// TODO Remove the test in elasticsearch 2.0.0
608+
if (in.getVersion().onOrAfter(Version.V_1_5_0)) {
609+
ignoreMissing = in.readOptionalBoolean();
610+
}
611+
612+
final Timestamp timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
583613
final boolean hasParentField = in.readBoolean();
584614
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
585615
}

‎src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.lucene.document.NumericDocValuesField;
2525
import org.apache.lucene.index.IndexOptions;
2626
import org.elasticsearch.Version;
27+
import org.elasticsearch.action.TimestampParsingException;
2728
import org.elasticsearch.common.Explicit;
2829
import org.elasticsearch.common.Nullable;
2930
import org.elasticsearch.common.Strings;
@@ -86,6 +87,7 @@ public static class Builder extends NumberFieldMapper.Builder<Builder, Timestamp
8687
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
8788
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
8889
private boolean explicitStore = false;
90+
private Boolean ignoreMissing = null;
8991

9092
public Builder() {
9193
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@@ -111,6 +113,11 @@ public Builder defaultTimestamp(String defaultTimestamp) {
111113
return builder;
112114
}
113115

116+
public Builder ignoreMissing(boolean ignoreMissing) {
117+
this.ignoreMissing = ignoreMissing;
118+
return builder;
119+
}
120+
114121
@Override
115122
public Builder store(boolean store) {
116123
explicitStore = true;
@@ -124,6 +131,7 @@ public TimestampFieldMapper build(BuilderContext context) {
124131
fieldType.setStored(false);
125132
}
126133
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp,
134+
ignoreMissing,
127135
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
128136
}
129137
}
@@ -133,6 +141,8 @@ public static class TypeParser implements Mapper.TypeParser {
133141
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
134142
TimestampFieldMapper.Builder builder = timestamp();
135143
parseField(builder, builder.name, node, parserContext);
144+
boolean defaultSet = false;
145+
Boolean ignoreMissing = null;
136146
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
137147
Map.Entry<String, Object> entry = iterator.next();
138148
String fieldName = Strings.toUnderscoreCase(entry.getKey());
@@ -148,10 +158,33 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
148158
builder.dateTimeFormatter(parseDateTimeFormatter(fieldNode.toString()));
149159
iterator.remove();
150160
} else if (fieldName.equals("default")) {
151-
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
161+
if (fieldNode == null) {
162+
if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_4_0_Beta1) &&
163+
parserContext.indexVersionCreated().before(Version.V_1_5_0)) {
164+
// We are reading an index created in 1.4 with feature #7036
165+
// `default: null` was explicitly set. We need to change this index to
166+
// `ignore_missing: false`
167+
builder.ignoreMissing(false);
168+
} else {
169+
throw new TimestampParsingException("default timestamp can not be set to null");
170+
}
171+
} else {
172+
builder.defaultTimestamp(fieldNode.toString());
173+
defaultSet = true;
174+
}
175+
iterator.remove();
176+
} else if (fieldName.equals("ignore_missing")) {
177+
ignoreMissing = nodeBooleanValue(fieldNode);
178+
builder.ignoreMissing(ignoreMissing);
152179
iterator.remove();
153180
}
154181
}
182+
183+
// We can not accept a default value and rejecting null values at the same time
184+
if (defaultSet && (ignoreMissing != null && ignoreMissing == false)) {
185+
throw new TimestampParsingException("default timestamp can not be set with ignore_missing set to false");
186+
}
187+
155188
return builder;
156189
}
157190
}
@@ -165,14 +198,16 @@ private static FieldType defaultFieldType(Settings settings) {
165198
private final String path;
166199
private final String defaultTimestamp;
167200
private final FieldType defaultFieldType;
201+
private final Boolean ignoreMissing;
168202

169203
public TimestampFieldMapper(Settings indexSettings) {
170204
this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
171-
Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
205+
null, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
172206
}
173207

174208
protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
175209
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp,
210+
Boolean ignoreMissing,
176211
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
177212
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
178213
@Nullable Settings fieldDataSettings, Settings indexSettings) {
@@ -185,6 +220,7 @@ protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAt
185220
this.path = path;
186221
this.defaultTimestamp = defaultTimestamp;
187222
this.defaultFieldType = defaultFieldType(indexSettings);
223+
this.ignoreMissing = ignoreMissing;
188224
}
189225

190226
@Override
@@ -204,6 +240,10 @@ public String defaultTimestamp() {
204240
return this.defaultTimestamp;
205241
}
206242

243+
public Boolean ignoreMissing() {
244+
return this.ignoreMissing;
245+
}
246+
207247
public FormatDateTimeFormatter dateTimeFormatter() {
208248
return this.dateTimeFormatter;
209249
}
@@ -292,6 +332,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
292332
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
293333
builder.field("default", defaultTimestamp);
294334
}
335+
if (includeDefaults || ignoreMissing != null) {
336+
builder.field("ignore_missing", ignoreMissing);
337+
}
295338
if (customFieldDataSettings != null) {
296339
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
297340
} else if (includeDefaults) {

0 commit comments

Comments
 (0)