Add support for a 'format' option in fields retrieval. (#57855)

The new `format` option allows for passing a custom date format:

```
POST logs-*/_search
{
  "fields": [
    "file.*",
    {
      "field": "event.timestamp",
      "format": "epoch_millis"
    },
    ...
  ]
}
```

Other API notes:
* We use the same syntax as `docvalue_fields` for consistency. Under the hood,
both `fields` and `docvalue_fields` use the same `FieldAndFormat` object to
share serialization logic.
* Only `date` and `date_range` fields support formatting currently.

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java

@@ -191,7 +191,10 @@ private Float objectToFloat(Object value) {
     }
 
     @Override
-    protected Float parseSourceValue(Object value) {
+    protected Float parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return objectToFloat(value);
     }
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java

@@ -169,7 +169,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value;
     }
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -497,7 +497,11 @@ private static double objectToDouble(Object value) {
     }
 
     @Override
-    protected Double parseSourceValue(Object value) {
+    protected Double parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
+
         double doubleValue = objectToDouble(value);
         double scalingFactor = fieldType().getScalingFactor();
         return Math.round(doubleValue * scalingFactor) / scalingFactor;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -492,7 +492,7 @@ protected void parseCreateField(ParseContext context) {
         }
 
         @Override
-        protected Object parseSourceValue(Object value) {
+        protected Object parseSourceValue(Object value, String format) {
             throw new UnsupportedOperationException();
         }
 
@@ -538,7 +538,7 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
         }
 
         @Override
-        protected Object parseSourceValue(Object value) {
+        protected Object parseSourceValue(Object value, String format) {
             throw new UnsupportedOperationException();
         }
 
@@ -695,7 +695,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value) {
+    protected String parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value.toString();
     }
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java

@@ -159,7 +159,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value) {
+    protected String parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
+
         return value.toString();
     }
 

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -194,7 +194,7 @@ public void testParseSourceValue() {
         Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
         RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);
 
-        assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001);
-        assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001);
+        assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001);
+        assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001);
     }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -409,7 +409,7 @@ public void testParseSourceValue() {
             .scalingFactor(100)
             .build(context);
 
-        assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001);
-        assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001);
+        assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001);
+        assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001);
     }
 }

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java

@@ -145,7 +145,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
         throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
     }
 

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -195,7 +195,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
         throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
     }
 

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -357,7 +357,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value;
     }
 

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -386,7 +386,10 @@ public void parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value;
     }
 

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -754,7 +754,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value) {
+    protected String parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value.toString();
     }
 }

plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

@@ -491,8 +491,8 @@ public void testParseSourceValue() {
         Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
         ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context);
 
-        assertEquals("value", mapper.parseSourceValue("value"));
-        assertEquals("42", mapper.parseSourceValue(42L));
-        assertEquals("true", mapper.parseSourceValue(true));
+        assertEquals("value", mapper.parseSourceValue("value", null));
+        assertEquals("42", mapper.parseSourceValue(42L, null));
+        assertEquals("true", mapper.parseSourceValue(true, null));
     }
 }

plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java

@@ -596,7 +596,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value) {
+    protected String parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value.toString();
     }
 

plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java

@@ -685,8 +685,8 @@ public void testParseSourceValue() {
             .searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer())
             .build(context);
 
-        assertEquals("value", mapper.parseSourceValue("value"));
-        assertEquals("42", mapper.parseSourceValue(42L));
-        assertEquals("true", mapper.parseSourceValue(true));
+        assertEquals("value", mapper.parseSourceValue("value", null));
+        assertEquals("42", mapper.parseSourceValue(42L, null));
+        assertEquals("true", mapper.parseSourceValue(true, null));
     }
 }

plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java

@@ -157,7 +157,10 @@ protected void parseCreateField(ParseContext context)
     }
 
     @Override
-    protected String parseSourceValue(Object value) {
+    protected String parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value.toString();
     }
 

rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -46,6 +46,53 @@ setup:
   - match: { hits.hits.0.fields.integer_range.0.lte: 42 }
 
 ---
+"Test date formatting":
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index.number_of_shards: 1
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              date:
+                type: date
+
+  - do:
+      index:
+        index:  test
+        id:     1
+        body:
+          keyword: "value"
+          date: "1990-12-29T22:30:00.000Z"
+
+  - do:
+      indices.refresh:
+        index: [ test ]
+
+  - do:
+      search:
+        index: test
+        body:
+          fields:
+            - field: date
+              format: "yyyy/MM/dd"
+
+  - is_true: hits.hits.0._id
+  - is_true: hits.hits.0._source
+  - match: { hits.hits.0.fields.date.0: "1990/12/29" }
+
+  - do:
+      catch: bad_request
+      search:
+        index: test
+        body:
+          fields:
+            - field: keyword
+              format: "yyyy/MM/dd"
+---
 "Test disable source":
   - do:
       indices.create:

server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -303,8 +303,24 @@ public SearchRequestBuilder addDocValueField(String name) {
         return addDocValueField(name, null);
     }
 
+    /**
+     * Adds a field to load and return. The field must be present in the document _source.
+     *
+     * @param name The field to load
+     */
     public SearchRequestBuilder addFetchField(String name) {
-        sourceBuilder().fetchField(name);
+        sourceBuilder().fetchField(name, null);
+        return this;
+    }
+
+    /**
+     * Adds a field to load and return. The field must be present in the document _source.
+     *
+     * @param name The field to load
+     * @param format TODO(jtibs): fill this in
+     */
+    public SearchRequestBuilder addFetchField(String name, String format) {
+        sourceBuilder().fetchField(name, format);
         return this;
     }
 

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -142,7 +142,7 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
         throw new UnsupportedOperationException();
     }
 

server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java

@@ -208,7 +208,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value) {
+    protected Object parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
         return value;
     }
 

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -262,7 +262,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    public Boolean parseSourceValue(Object value) {
+    public Boolean parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
+
         if (value instanceof Boolean) {
             return (Boolean) value;
         } else {

server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java

@@ -624,7 +624,11 @@ private void parse(ParseContext parseContext, Token token,
     }
 
     @Override
-    protected List<?> parseSourceValue(Object value) {
+    protected List<?> parseSourceValue(Object value, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
+
         if (value instanceof List) {
             return (List<?>) value;
         } else {

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -628,12 +628,16 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    public String parseSourceValue(Object value) {
+    public String parseSourceValue(Object value, String format) {
         String date = value.toString();
         long timestamp = fieldType().parse(date);
-
         ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
-        return fieldType().dateTimeFormatter().format(dateTime);
+
+        DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter();
+        if (format != null) {
+            dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale());
+        }
+        return dateTimeFormatter.format(dateTime);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -24,6 +24,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -280,21 +281,22 @@ public void parse(ParseContext context) throws IOException {
      * Some mappers may need more flexibility and can override this entire method instead.
      *
      * @param lookup a lookup structure over the document's source.
+     * @param format an optional format string used when formatting values, for example a date format.
      * @return a list a standardized field values.
      */
-    public List<?> lookupValues(SourceLookup lookup) {
+    public List<?> lookupValues(SourceLookup lookup, @Nullable String format) {
         Object sourceValue = lookup.extractValue(name());
         if (sourceValue == null) {
             return List.of();
         }
 
         List<Object> values = new ArrayList<>();
         if (parsesArrayValue()) {
-            return (List<?>) parseSourceValue(sourceValue);
+            return (List<?>) parseSourceValue(sourceValue, format);
         } else {
             List<?> sourceValues = sourceValue instanceof List ? (List<?>) sourceValue : List.of(sourceValue);
             for (Object value : sourceValues) {
-                Object parsedValue = parseSourceValue(value);
+                Object parsedValue = parseSourceValue(value, format);
                 values.add(parsedValue);
             }
         }
@@ -308,7 +310,7 @@ public List<?> lookupValues(SourceLookup lookup) {
      *
      * Note that when overriding this method, {@link #lookupValues} should *not* be overridden.
      */
-    protected abstract Object parseSourceValue(Object value);
+    protected abstract Object parseSourceValue(Object value, @Nullable String format);
 
     protected void createFieldNamesField(ParseContext context) {
         FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType();