Skip to content

Commit 7a444df

Browse files
romseygeekromseygeek
committed
Move index analyzer management to FieldMapper/MapperService (elastic#63937)
Index-time analyzers are currently specified on the MappedFieldType. This has a number of unfortunate consequences; for example, field mappers that index data into implementation sub-fields, such as prefix or phrase accelerators on text fields, need to expose these sub-fields as MappedFieldTypes, which means that they then appear in field caps, are externally searchable, etc. It also adds index-time logic to a class that should only be concerned with search-time behaviour. This commit removes references to the index analyzer from MappedFieldType. Instead, FieldMappers that use the terms index can pass either a single analyzer or a Map of fields to analyzers to their super constructor, which are then exposed via a new FieldMapper#indexAnalyzers() method; all index-time analysis is mediated through the delegating analyzer wrapper on MapperService. In a follow-up, this will make it possible to register multiple field analyzers from a single FieldMapper, removing the need for 'hidden' mapper implementations on text field, parent joins, and elsewhere.
1 parent 1091d1b commit 7a444df

File tree

53 files changed

+457
-381
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+457
-381
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@ public static final class RankFeatureFieldType extends MappedFieldType {
9191
public RankFeatureFieldType(String name, Map<String, String> meta, boolean positiveScoreImpact) {
9292
super(name, true, false, false, TextSearchInfo.NONE, meta);
9393
this.positiveScoreImpact = positiveScoreImpact;
94-
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
9594
}
9695

9796
@Override
@@ -136,7 +135,7 @@ public Query termQuery(Object value, QueryShardContext context) {
136135

137136
private RankFeatureFieldMapper(String simpleName, MappedFieldType mappedFieldType,
138137
MultiFields multiFields, CopyTo copyTo, boolean positiveScoreImpact) {
139-
super(simpleName, mappedFieldType, multiFields, copyTo);
138+
super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
140139
this.positiveScoreImpact = positiveScoreImpact;
141140
}
142141

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ public static final class RankFeaturesFieldType extends MappedFieldType {
6868

6969
public RankFeaturesFieldType(String name, Map<String, String> meta) {
7070
super(name, false, false, false, TextSearchInfo.NONE, meta);
71-
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
7271
}
7372

7473
@Override
@@ -99,7 +98,7 @@ public Query termQuery(Object value, QueryShardContext context) {
9998

10099
private RankFeaturesFieldMapper(String simpleName, MappedFieldType mappedFieldType,
101100
MultiFields multiFields, CopyTo copyTo) {
102-
super(simpleName, mappedFieldType, multiFields, copyTo);
101+
super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
103102
}
104103

105104
@Override

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java‎

Lines changed: 29 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import java.util.ArrayList;
5858
import java.util.Arrays;
5959
import java.util.Collections;
60+
import java.util.HashMap;
6061
import java.util.Iterator;
6162
import java.util.List;
6263
import java.util.Map;
@@ -95,18 +96,14 @@ public static class Defaults {
9596
public static final TypeParser PARSER
9697
= new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
9798

98-
private static SearchAsYouTypeFieldMapper toType(FieldMapper in) {
99-
return (SearchAsYouTypeFieldMapper) in;
100-
}
101-
102-
private static SearchAsYouTypeFieldType ft(FieldMapper in) {
103-
return toType(in).fieldType();
99+
private static Builder builder(FieldMapper in) {
100+
return ((SearchAsYouTypeFieldMapper)in).builder;
104101
}
105102

106103
public static class Builder extends FieldMapper.Builder {
107104

108-
private final Parameter<Boolean> index = Parameter.indexParam(m -> toType(m).index, true);
109-
private final Parameter<Boolean> store = Parameter.storeParam(m -> toType(m).store, false);
105+
private final Parameter<Boolean> index = Parameter.indexParam(m -> builder(m).index.get(), true);
106+
private final Parameter<Boolean> store = Parameter.storeParam(m -> builder(m).store.get(), false);
110107

111108
// This is only here because for some reason the initial impl of this always serialized
112109
// `doc_values=false`, even though it cannot be set; and so we need to continue
@@ -120,7 +117,7 @@ public static class Builder extends FieldMapper.Builder {
120117
.alwaysSerialize();
121118

122119
private final Parameter<Integer> maxShingleSize = Parameter.intParam("max_shingle_size", false,
123-
m -> toType(m).maxShingleSize, Defaults.MAX_SHINGLE_SIZE)
120+
m -> builder(m).maxShingleSize.get(), Defaults.MAX_SHINGLE_SIZE)
124121
.setValidator(v -> {
125122
if (v < MAX_SHINGLE_SIZE_LOWER_BOUND || v > MAX_SHINGLE_SIZE_UPPER_BOUND) {
126123
throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND
@@ -130,17 +127,17 @@ public static class Builder extends FieldMapper.Builder {
130127
.alwaysSerialize();
131128

132129
final TextParams.Analyzers analyzers;
133-
final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> ft(m).getTextSearchInfo().getSimilarity());
130+
final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> builder(m).similarity.get());
134131

135-
final Parameter<String> indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions);
136-
final Parameter<Boolean> norms = TextParams.norms(true, m -> ft(m).getTextSearchInfo().hasNorms());
137-
final Parameter<String> termVectors = TextParams.termVectors(m -> toType(m).termVectors);
132+
final Parameter<String> indexOptions = TextParams.indexOptions(m -> builder(m).indexOptions.get());
133+
final Parameter<Boolean> norms = TextParams.norms(true, m -> builder(m).norms.get());
134+
final Parameter<String> termVectors = TextParams.termVectors(m -> builder(m).termVectors.get());
138135

139136
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
140137

141138
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
142139
super(name);
143-
this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
140+
this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers);
144141
}
145142

146143
@Override
@@ -159,12 +156,15 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
159156
fieldType.setStored(store.getValue());
160157
TextParams.setTermVectorParams(termVectors.getValue(), fieldType);
161158

159+
Map<String, NamedAnalyzer> indexAnalyzers = new HashMap<>();
160+
162161
NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
163162
NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
164163

165164
SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity.getValue(),
166165
analyzers.getSearchAnalyzer(), analyzers.getSearchQuoteAnalyzer(), meta.getValue());
167-
ft.setIndexAnalyzer(analyzers.getIndexAnalyzer());
166+
167+
indexAnalyzers.put(ft.name(), indexAnalyzer);
168168

169169
// set up the prefix field
170170
FieldType prefixft = new FieldType(fieldType);
@@ -182,8 +182,10 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
182182
TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity.getValue(), prefixSearchWrapper, searchAnalyzer);
183183
final PrefixFieldType prefixFieldType
184184
= new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM);
185-
prefixFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper));
185+
final NamedAnalyzer prefixAnalyzer
186+
= new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper);
186187
final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType);
188+
indexAnalyzers.put(prefixFieldType.name(), prefixAnalyzer);
187189

188190
// set up the shingle fields
189191
final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize.getValue() - 1];
@@ -203,14 +205,16 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
203205
TextSearchInfo textSearchInfo
204206
= new TextSearchInfo(shingleft, similarity.getValue(), shingleSearchWrapper, shingleSearchQuoteWrapper);
205207
final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo);
206-
shingleFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper));
207208
shingleFieldType.setPrefixFieldType(prefixFieldType);
208209
shingleFieldTypes[i] = shingleFieldType;
210+
NamedAnalyzer shingleAnalyzer
211+
= new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper);
209212
shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType);
213+
indexAnalyzers.put(shingleFieldType.name(), shingleAnalyzer);
210214
}
211215
ft.setPrefixField(prefixFieldType);
212216
ft.setShingleFields(shingleFieldTypes);
213-
return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this);
217+
return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), indexAnalyzers, prefixFieldMapper, shingleFieldMappers, this);
214218
}
215219
}
216220

@@ -546,29 +550,23 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
546550
}
547551
}
548552

549-
private final boolean index;
550-
private final boolean store;
551-
private final String indexOptions;
552-
private final String termVectors;
553-
554553
private final int maxShingleSize;
555554
private final PrefixFieldMapper prefixField;
556555
private final ShingleFieldMapper[] shingleFields;
556+
private final Builder builder;
557557

558558
public SearchAsYouTypeFieldMapper(String simpleName,
559559
SearchAsYouTypeFieldType mappedFieldType,
560560
CopyTo copyTo,
561+
Map<String, NamedAnalyzer> indexAnalyzers,
561562
PrefixFieldMapper prefixField,
562563
ShingleFieldMapper[] shingleFields,
563564
Builder builder) {
564-
super(simpleName, mappedFieldType, MultiFields.empty(), copyTo);
565+
super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo);
565566
this.prefixField = prefixField;
566567
this.shingleFields = shingleFields;
567568
this.maxShingleSize = builder.maxShingleSize.getValue();
568-
this.index = builder.index.getValue();
569-
this.store = builder.store.getValue();
570-
this.indexOptions = builder.indexOptions.getValue();
571-
this.termVectors = builder.termVectors.getValue();
569+
this.builder = builder;
572570
}
573571

574572
@Override
@@ -578,12 +576,12 @@ protected void parseCreateField(ParseContext context) throws IOException {
578576
return;
579577
}
580578

581-
if (this.index == false && this.store == false) {
579+
if (this.builder.index.get() == false && this.builder.store.get() == false) {
582580
return;
583581
}
584582

585583
context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType));
586-
if (this.index) {
584+
if (this.builder.index.get()) {
587585
for (ShingleFieldMapper subFieldMapper : shingleFields) {
588586
context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType()));
589587
}
@@ -599,9 +597,8 @@ protected String contentType() {
599597
return CONTENT_TYPE;
600598
}
601599

602-
@Override
603600
public FieldMapper.Builder getMergeBuilder() {
604-
return new Builder(simpleName(), () -> fieldType().indexAnalyzer()).init(this);
601+
return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
605602
}
606603

607604
public static String getShingleFieldName(String parentField, int shingleSize) {

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java‎

Lines changed: 34 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
import java.util.Collection;
6363
import java.util.Collections;
6464
import java.util.List;
65+
import java.util.Map;
6566
import java.util.Set;
6667
import java.util.stream.Collectors;
6768
import java.util.stream.Stream;
@@ -180,12 +181,12 @@ public void testDefaultConfiguration() throws IOException {
180181
assertRootFieldMapper(rootMapper, 3, "default");
181182

182183
PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix");
183-
assertPrefixFieldType(prefixFieldMapper.fieldType(), 3, "default");
184+
assertPrefixFieldType(prefixFieldMapper, rootMapper.indexAnalyzers(), 3, "default");
184185

185-
assertShingleFieldType(
186-
getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, "default", prefixFieldMapper.fieldType());
187-
assertShingleFieldType(
188-
getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, "default", prefixFieldMapper.fieldType());
186+
assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._2gram"),
187+
rootMapper.indexAnalyzers(), 2, "default", prefixFieldMapper.fieldType());
188+
assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._3gram"),
189+
rootMapper.indexAnalyzers(), 3, "default", prefixFieldMapper.fieldType());
189190
}
190191

191192
public void testConfiguration() throws IOException {
@@ -201,14 +202,14 @@ public void testConfiguration() throws IOException {
201202
assertRootFieldMapper(rootMapper, maxShingleSize, analyzerName);
202203

203204
PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix");
204-
assertPrefixFieldType(prefixFieldMapper.fieldType(), maxShingleSize, analyzerName);
205+
assertPrefixFieldType(prefixFieldMapper, rootMapper.indexAnalyzers(), maxShingleSize, analyzerName);
205206

206-
assertShingleFieldType(
207-
getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, analyzerName, prefixFieldMapper.fieldType());
208-
assertShingleFieldType(
209-
getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, analyzerName, prefixFieldMapper.fieldType());
210-
assertShingleFieldType(
211-
getShingleFieldMapper(defaultMapper, "field._4gram").fieldType(), 4, analyzerName, prefixFieldMapper.fieldType());
207+
assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._2gram"),
208+
rootMapper.indexAnalyzers(), 2, analyzerName, prefixFieldMapper.fieldType());
209+
assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._3gram"),
210+
rootMapper.indexAnalyzers(), 3, analyzerName, prefixFieldMapper.fieldType());
211+
assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._4gram"),
212+
rootMapper.indexAnalyzers(), 4, analyzerName, prefixFieldMapper.fieldType());
212213
}
213214

214215
public void testSimpleMerge() throws IOException {
@@ -600,47 +601,53 @@ private static void assertRootFieldMapper(SearchAsYouTypeFieldMapper mapper,
600601

601602
assertThat(mapper.maxShingleSize(), equalTo(maxShingleSize));
602603
assertThat(mapper.fieldType(), notNullValue());
603-
assertSearchAsYouTypeFieldType(mapper.fieldType(), maxShingleSize, analyzerName, mapper.prefixField().fieldType());
604+
assertSearchAsYouTypeFieldType(mapper, mapper.fieldType(), maxShingleSize, analyzerName, mapper.prefixField().fieldType());
604605

605606
assertThat(mapper.prefixField(), notNullValue());
606607
assertThat(mapper.prefixField().fieldType().parentField, equalTo(mapper.name()));
607-
assertPrefixFieldType(mapper.prefixField().fieldType(), maxShingleSize, analyzerName);
608+
assertPrefixFieldType(mapper.prefixField(), mapper.indexAnalyzers, maxShingleSize, analyzerName);
608609

609610

610611
for (int shingleSize = 2; shingleSize <= maxShingleSize; shingleSize++) {
611612
final ShingleFieldMapper shingleFieldMapper = mapper.shingleFields()[shingleSize - 2];
612613
assertThat(shingleFieldMapper, notNullValue());
613-
assertShingleFieldType(shingleFieldMapper.fieldType(), shingleSize, analyzerName, mapper.prefixField().fieldType());
614+
assertShingleFieldType(shingleFieldMapper, mapper.indexAnalyzers, shingleSize,
615+
analyzerName, mapper.prefixField().fieldType());
614616
}
615617

616618
final int numberOfShingleSubfields = (maxShingleSize - 2) + 1;
617619
assertThat(mapper.shingleFields().length, equalTo(numberOfShingleSubfields));
618620
}
619621

620-
private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldType fieldType, int maxShingleSize,
622+
private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldMapper mapper,
623+
SearchAsYouTypeFieldType fieldType,
624+
int maxShingleSize,
621625
String analyzerName,
622626
PrefixFieldType prefixFieldType) {
623627

624628
assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1));
625-
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
629+
NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().get(fieldType.name());
630+
for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) {
626631
assertThat(analyzer.name(), equalTo(analyzerName));
627632
}
628633
int shingleSize = 2;
629-
for (ShingleFieldType shingleField : fieldType.shingleFields) {
630-
assertShingleFieldType(shingleField, shingleSize++, analyzerName, prefixFieldType);
634+
for (ShingleFieldMapper shingleField : mapper.shingleFields()) {
635+
assertShingleFieldType(shingleField, mapper.indexAnalyzers(), shingleSize++, analyzerName, prefixFieldType);
631636
}
632637

633638
assertThat(fieldType.prefixField, equalTo(prefixFieldType));
634639
}
635640

636-
private static void assertShingleFieldType(ShingleFieldType fieldType,
641+
private static void assertShingleFieldType(ShingleFieldMapper mapper,
642+
Map<String, NamedAnalyzer> indexAnalyzers,
637643
int shingleSize,
638644
String analyzerName,
639645
PrefixFieldType prefixFieldType) {
640646

647+
ShingleFieldType fieldType = mapper.fieldType();
641648
assertThat(fieldType.shingleSize, equalTo(shingleSize));
642649

643-
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
650+
for (NamedAnalyzer analyzer : asList(indexAnalyzers.get(fieldType.name()), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
644651
assertThat(analyzer.name(), equalTo(analyzerName));
645652
if (shingleSize > 1) {
646653
final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer();
@@ -653,12 +660,15 @@ private static void assertShingleFieldType(ShingleFieldType fieldType,
653660

654661
}
655662

656-
private static void assertPrefixFieldType(PrefixFieldType fieldType, int shingleSize, String analyzerName) {
657-
for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) {
663+
private static void assertPrefixFieldType(PrefixFieldMapper mapper, Map<String, NamedAnalyzer> indexAnalyzers,
664+
int shingleSize, String analyzerName) {
665+
PrefixFieldType fieldType = mapper.fieldType();
666+
NamedAnalyzer indexAnalyzer = indexAnalyzers.get(fieldType.name());
667+
for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) {
658668
assertThat(analyzer.name(), equalTo(analyzerName));
659669
}
660670

661-
final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.indexAnalyzer().analyzer();
671+
final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) indexAnalyzer.analyzer();
662672
final SearchAsYouTypeAnalyzer wrappedSearchAnalyzer
663673
= (SearchAsYouTypeAnalyzer) fieldType.getTextSearchInfo().getSearchAnalyzer().analyzer();
664674
for (SearchAsYouTypeAnalyzer analyzer : asList(wrappedIndexAnalyzer, wrappedSearchAnalyzer)) {

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,6 @@ public void testPrefixQuery() {
113113

114114
public void testFetchSourceValue() throws IOException {
115115
SearchAsYouTypeFieldType fieldType = createFieldType();
116-
fieldType.setIndexAnalyzer(Lucene.STANDARD_ANALYZER);
117116

118117
assertEquals(org.elasticsearch.common.collect.List.of("value"), fetchSourceValue(fieldType, "value"));
119118
assertEquals(org.elasticsearch.common.collect.List.of("42"), fetchSourceValue(fieldType, 42L));

0 commit comments

Comments
 (0)