Allow field mappers to retrieve fields from source. (#56928)

This PR adds new method `FieldMapper#lookupValues(SourceLookup)` that extracts and parses the source values. This lets us return values like numbers and dates in a consistent format, and also handle special data types like `constant_keyword`. The `lookupValues` method calls into `parseSourceValue`, which mappers can override to specify how values should be parsed.
elastic · Jul 21, 2020 · db11a9e · db11a9e
1 parent ed67d6c
commit db11a9e
Show file tree

Hide file tree

Showing 58 changed files with 753 additions and 80 deletions.
diff --git a/...es/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/...es/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java
@@ -152,11 +152,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
         float value;
         if (context.externalValueSet()) {
             Object v = context.externalValue();
-            if (v instanceof Number) {
-                value = ((Number) v).floatValue();
-            } else {
-                value = Float.parseFloat(v.toString());
-            }
+            value = objectToFloat(v);
         } else if (context.parser().currentToken() == Token.VALUE_NULL) {
             // skip
             return;
@@ -176,6 +172,19 @@ protected void parseCreateField(ParseContext context) throws IOException {
         context.doc().addWithKey(name(), new FeatureField("_feature", name(), value));
     }
 
+    private Float objectToFloat(Object value) {
+        if (value instanceof Number) {
+            return ((Number) value).floatValue();
+        } else {
+            return Float.parseFloat(value.toString());
+        }
+    }
+
+    @Override
+    protected Float parseSourceValue(Object value) {
+        return objectToFloat(value);
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

diff --git a/...s/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/...s/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java
@@ -159,6 +159,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         throw new AssertionError("parse is implemented directly");
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        return value;
+    }
+
     @Override
     protected boolean indexedByDefault() {
         return false;

diff --git a/...es/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/...es/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
@@ -474,6 +474,13 @@ private static double objectToDouble(Object value) {
         return doubleValue;
     }
 
+    @Override
+    protected Double parseSourceValue(Object value) {
+        double doubleValue = objectToDouble(value);
+        double scalingFactor = fieldType().getScalingFactor();
+        return Math.round(doubleValue * scalingFactor) / scalingFactor;
+    }
+
     private static class ScaledFloatIndexFieldData extends IndexNumericFieldData {
 
         private final IndexNumericFieldData scaledFieldData;

diff --git a/...apper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/...apper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java
@@ -418,6 +418,11 @@ protected void parseCreateField(ParseContext context) {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        protected Object parseSourceValue(Object value) {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
@@ -459,6 +464,11 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
         }
 
+        @Override
+        protected Object parseSourceValue(Object value) {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         protected String contentType() {
             return "shingle";
@@ -577,6 +587,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
     }
 
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

diff --git a/...les/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/...les/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java
@@ -158,6 +158,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, indexed, docValued, stored));
     }
 
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
+
     /**
      * Count position increments in a token stream.  Package private for testing.
      * @param analyzer analyzer to create token stream

diff --git a/...pper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/...pper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java
@@ -23,9 +23,12 @@
 import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
 import org.apache.lucene.document.FeatureField;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
@@ -186,4 +189,12 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
                 e.getCause().getMessage());
     }
 
+    public void testParseSourceValue() {
+        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);
+
+        assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001);
+        assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001);
+    }
 }
diff --git a/...pper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/...pper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
@@ -21,9 +21,12 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
@@ -398,4 +401,15 @@ public void testMeta() throws Exception {
                 new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE);
         assertEquals(mapping3, mapper.mappingSource().toString());
     }
+
+    public void testParseSourceValue() {
+        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field")
+            .scalingFactor(100)
+            .build(context);
+
+        assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001);
+        assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001);
+    }
 }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java
@@ -135,6 +135,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         throw new IllegalStateException("Should never be called");
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java
@@ -185,6 +185,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
+    }
+
     @Override
     protected void mergeOptions(FieldMapper other, List<String> conflicts) {
         ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) other;

diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java
@@ -347,6 +347,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called");
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        return value;
+    }
+
     @Override
     public void parse(ParseContext context) throws IOException {
         context.path().add(simpleName());

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -367,6 +367,11 @@ public void parse(ParseContext context) throws IOException {
         processQuery(query, context);
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        return value;
+    }
+
     static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField,
                                         QueryBuilder queryBuilder, ParseContext context) throws IOException {
         try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {

diff --git a/...ysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/...ysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java
@@ -731,4 +731,9 @@ protected void parseCreateField(ParseContext context) throws IOException {
             createFieldNamesField(context);
         }
     }
+
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
 }
diff --git a/...icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/...icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java
@@ -26,9 +26,12 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
@@ -483,4 +486,13 @@ public void testUpdateIgnoreAbove() throws IOException {
         indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
     }
 
+    public void testParseSourceValue() {
+        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context);
+
+        assertEquals("value", mapper.parseSourceValue("value"));
+        assertEquals("42", mapper.parseSourceValue(42L));
+        assertEquals("true", mapper.parseSourceValue(true));
+    }
 }
diff --git a/.../src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/.../src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -583,6 +583,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
     }
 
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

diff --git a/...test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/...test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -28,10 +28,12 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -44,8 +46,11 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -672,4 +677,19 @@ public void testEmptyName() throws IOException {
         assertThat(e.getMessage(), containsString("name cannot be empty string"));
     }
 
+    public void testParseSourceValue() {
+        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+
+        FieldMapper fieldMapper = new AnnotatedTextFieldMapper.Builder("field")
+            .indexAnalyzer(indexService.getIndexAnalyzers().getDefaultIndexAnalyzer())
+            .searchAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchAnalyzer())
+            .searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer())
+            .build(context);
+        AnnotatedTextFieldMapper mapper = (AnnotatedTextFieldMapper) fieldMapper;
+
+        assertEquals("value", mapper.parseSourceValue("value"));
+        assertEquals("42", mapper.parseSourceValue(42L));
+        assertEquals("true", mapper.parseSourceValue(true));
+    }
 }
diff --git a/...pper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/...pper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
@@ -147,6 +147,11 @@ protected void parseCreateField(ParseContext context)
         }
     }
 
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
+
     @Override
     protected boolean indexedByDefault() {
         return false;
@@ -156,5 +161,4 @@ protected boolean indexedByDefault() {
     protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
     }
-
 }