Skip to content

Commit db144f4

Browse files
nknizenknize
committed
Override doc_value parameter in Spatial XPack module
This commit enables parsing the doc_value parameter in GeoShapeFieldMapper and ShapeFieldMapper when the xpack spatial module is loaded. Licensing is Basic+. This commit introduces the following behavior: * doc_values defaults to true for GeoShapeFieldMapper and ShapeFieldMapper when the Spatial XPack module is available * doc_values are not supported for geo_shape PrefixTree indexing (same behavior as before) * doc_values defaults to false for GeoShapeFieldMapper and ShapeFieldMapper when the Spatial XPack module is not available (e.g., OSS) * when doc_values are set in OSS, a MapperParsingException is thrown (same behavior as before)
1 parent 51adeaa commit db144f4

File tree

18 files changed

+532
-64
lines changed

18 files changed

+532
-64
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ public static class Defaults {
6969
public static final Explicit<Boolean> IGNORE_Z_VALUE = new Explicit<>(true, false);
7070
}
7171

72+
protected static List<ParserHandler> PARSER_EXTENSIONS = new ArrayList<>();
7273

7374
/**
7475
* Interface representing an preprocessor in geo-shape indexing pipeline
@@ -80,6 +81,8 @@ public interface Indexer<Parsed, Processed> {
8081
Class<Processed> processedClass();
8182

8283
List<IndexableField> indexShape(ParseContext context, Processed shape);
84+
85+
void indexDocValues(ParseContext context, Processed shape);
8386
}
8487

8588
/**
@@ -222,6 +225,7 @@ protected Builder newBuilder(String name, Map<String, Object> params) {
222225
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
223226
Map<String, Object> params = new HashMap<>();
224227
boolean parsedDeprecatedParameters = false;
228+
boolean parsedExtension = false;
225229
params.put(DEPRECATED_PARAMETERS_KEY, new DeprecatedParameters());
226230
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
227231
Map.Entry<String, Object> entry = iterator.next();
@@ -245,6 +249,13 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
245249
XContentMapValues.nodeBooleanValue(fieldNode,
246250
name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()));
247251
iterator.remove();
252+
} else if (PARSER_EXTENSIONS.isEmpty() == false) {
253+
for (ParserHandler parser : PARSER_EXTENSIONS) {
254+
if (parser.parse(iterator, fieldName, fieldNode, params, parserContext.indexVersionCreated())) {
255+
parsedExtension = true;
256+
break;
257+
}
258+
}
248259
}
249260
}
250261
if (parsedDeprecatedParameters == false) {
@@ -268,6 +279,12 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
268279
builder.orientation((Orientation)params.get(Names.ORIENTATION.getPreferredName()));
269280
}
270281

282+
if (parsedExtension) {
283+
for (ParserHandler parser : PARSER_EXTENSIONS) {
284+
parser.config(params, builder);
285+
}
286+
}
287+
271288
return builder;
272289
}
273290
}
@@ -399,6 +416,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
399416
if (includeDefaults || ignoreZValue.explicit()) {
400417
builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value());
401418
}
419+
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
420+
doXContentDocValues(builder, includeDefaults);
421+
}
402422
}
403423

404424
public Explicit<Boolean> coerce() {
@@ -440,6 +460,9 @@ public void parse(ParseContext context) throws IOException {
440460
for (IndexableField field : fields) {
441461
context.doc().add(field);
442462
}
463+
if (fieldType.hasDocValues()) {
464+
geometryIndexer.indexDocValues(context, shape);
465+
}
443466
} catch (Exception e) {
444467
if (ignoreMalformed.value() == false) {
445468
throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(),
@@ -449,4 +472,19 @@ public void parse(ParseContext context) throws IOException {
449472
}
450473
}
451474

475+
public static void registerParserExtensions(List<ParserHandler> parserExtensions) {
476+
PARSER_EXTENSIONS.addAll(parserExtensions);
477+
}
478+
479+
public interface ParserHandler {
480+
boolean parse(Iterator<Map.Entry<String, Object>> iterator, String name, Object node, Map<String, Object> params,
481+
Version indexCreatedVersion);
482+
@SuppressWarnings("rawtypes")
483+
void config(Map<String, Object> params, Builder builder);
484+
}
485+
486+
public interface ParserExtension {
487+
List<ParserHandler> getParserExtensions();
488+
}
489+
452490
}

‎server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@
1919
package org.elasticsearch.index.mapper;
2020

2121
import org.apache.lucene.document.LatLonShape;
22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.common.Explicit;
2324
import org.elasticsearch.common.geo.GeometryParser;
2425
import org.elasticsearch.common.geo.builders.ShapeBuilder;
2526
import org.elasticsearch.common.settings.Settings;
2627
import org.elasticsearch.geometry.Geometry;
2728
import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor;
2829

30+
import java.util.HashMap;
31+
import java.util.Map;
32+
2933
/**
3034
* FieldMapper for indexing {@link LatLonShape}s.
3135
* <p>
@@ -49,9 +53,18 @@
4953
public class GeoShapeFieldMapper extends AbstractGeometryFieldMapper<Geometry, Geometry> {
5054
public static final String CONTENT_TYPE = "geo_shape";
5155

56+
public static Map<String, DataHandlerFactory> DATA_HANDLER_FACTORIES = new HashMap<>();
57+
58+
public static class Defaults extends AbstractGeometryFieldMapper.Defaults {
59+
public static final Explicit<String> DATA_HANDLER = new Explicit<>("default", false);
60+
}
61+
5262
public static class Builder extends AbstractGeometryFieldMapper.Builder<AbstractGeometryFieldMapper.Builder, GeoShapeFieldMapper> {
63+
DataHandler dataHandler;
64+
5365
public Builder(String name) {
5466
super (name, new GeoShapeFieldType(), new GeoShapeFieldType());
67+
this.dataHandler = resolveDataHandler(Defaults.DATA_HANDLER.value());
5568
}
5669

5770
@Override
@@ -61,18 +74,24 @@ public GeoShapeFieldMapper build(BuilderContext context) {
6174
ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
6275
}
6376

77+
@Override
78+
protected boolean defaultDocValues(Version indexCreated) {
79+
return dataHandler.defaultDocValues(indexCreated);
80+
}
81+
6482
@Override
6583
protected void setupFieldType(BuilderContext context) {
6684
super.setupFieldType(context);
6785

6886
GeoShapeFieldType fieldType = (GeoShapeFieldType)fieldType();
6987
boolean orientation = fieldType.orientation == ShapeBuilder.Orientation.RIGHT;
7088

89+
// @todo the GeometryParser can be static since it doesn't hold state?
7190
GeometryParser geometryParser = new GeometryParser(orientation, coerce(context).value(), ignoreZValue().value());
72-
73-
fieldType.setGeometryIndexer(new GeoShapeIndexer(orientation, fieldType.name()));
7491
fieldType.setGeometryParser( (parser, mapper) -> geometryParser.parse(parser));
75-
fieldType.setGeometryQueryBuilder(new VectorGeoShapeQueryProcessor());
92+
93+
fieldType.setGeometryIndexer(dataHandler.newIndexer(orientation, fieldType));
94+
fieldType.setGeometryQueryBuilder(dataHandler.newQueryProcessor());
7695
}
7796
}
7897

@@ -124,4 +143,49 @@ public GeoShapeFieldType fieldType() {
124143
protected String contentType() {
125144
return CONTENT_TYPE;
126145
}
146+
147+
public static void registerDataHandlers(Map<String, DataHandlerFactory> dataHandlerFactories) {
148+
DATA_HANDLER_FACTORIES.putAll(dataHandlerFactories);
149+
}
150+
151+
public static DataHandler resolveDataHandler(String handlerKey) {
152+
if (DATA_HANDLER_FACTORIES.containsKey(handlerKey)) {
153+
return DATA_HANDLER_FACTORIES.get(handlerKey).newDataHandler();
154+
}
155+
throw new IllegalArgumentException("dataHandler [" + handlerKey + "] not supported");
156+
}
157+
158+
public interface DataHandlerFactory {
159+
DataHandler newDataHandler();
160+
}
161+
162+
public abstract static class DataHandler {
163+
public abstract Indexer newIndexer(boolean orientation, MappedFieldType fieldType);
164+
public abstract QueryProcessor newQueryProcessor();
165+
public abstract boolean defaultDocValues(Version indexCreatedVersion);
166+
}
167+
168+
static {
169+
DATA_HANDLER_FACTORIES.put(Defaults.DATA_HANDLER.value(), () ->
170+
new DataHandler() {
171+
@Override
172+
public boolean defaultDocValues(Version indexCreatedVersion) {
173+
return false;
174+
}
175+
176+
@Override
177+
public Indexer newIndexer(boolean orientation, MappedFieldType fieldType) {
178+
return new GeoShapeIndexer(orientation, fieldType.name());
179+
}
180+
181+
@Override
182+
public QueryProcessor newQueryProcessor() {
183+
return new VectorGeoShapeQueryProcessor();
184+
}
185+
});
186+
}
187+
188+
public interface Extension {
189+
Map<String, DataHandlerFactory> getDataHandlerFactories();
190+
}
127191
}

‎server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
/**
4747
* Utility class that converts geometries into Lucene-compatible form for indexing in a geo_shape field.
4848
*/
49-
public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
49+
public class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
5050

5151
private final boolean orientation;
5252
private final String name;
@@ -185,6 +185,11 @@ public List<IndexableField> indexShape(ParseContext context, Geometry shape) {
185185
return visitor.fields();
186186
}
187187

188+
@Override
189+
public void indexDocValues(ParseContext context, Geometry shape) {
190+
throw new UnsupportedOperationException("DocValues are not supported");
191+
}
192+
188193
private static class LuceneGeometryIndexer implements GeometryVisitor<Void, RuntimeException> {
189194
private List<IndexableField> fields = new ArrayList<>();
190195
private String name;

‎server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,11 @@ private void setupPrefixTrees() {
264264
protected void setupFieldType(BuilderContext context) {
265265
super.setupFieldType(context);
266266

267+
if (context.indexCreatedVersion().onOrAfter(Version.V_8_0_0) && fieldType().hasDocValues()) {
268+
throw new ElasticsearchParseException("Field parameter [{}] is not supported for [{}] field type while using prefix trees",
269+
TypeParsers.DOC_VALUES, CONTENT_TYPE);
270+
}
271+
267272
fieldType().setGeometryIndexer(new LegacyGeoShapeIndexer(fieldType()));
268273
fieldType().setGeometryParser(ShapeParser::parse);
269274
fieldType().setGeometryQueryBuilder(new LegacyGeoShapeQueryProcessor(fieldType()));

‎server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,9 @@ public List<IndexableField> indexShape(ParseContext context, Shape shape) {
6969
}
7070
return Arrays.asList(fieldType.defaultPrefixTreeStrategy().createIndexableFields(shape));
7171
}
72+
73+
@Override
74+
public void indexDocValues(ParseContext context, Shape shape) {
75+
throw new UnsupportedOperationException("[" + fieldType.name() + "] does not support doc_values");
76+
}
7277
}

‎server/src/main/java/org/elasticsearch/indices/IndicesModule.java‎

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import org.elasticsearch.indices.mapper.MapperRegistry;
6666
import org.elasticsearch.indices.store.IndicesStore;
6767
import org.elasticsearch.plugins.MapperPlugin;
68+
import org.elasticsearch.plugins.Plugin;
6869

6970
import java.util.Arrays;
7071
import java.util.Collection;
@@ -73,6 +74,7 @@
7374
import java.util.List;
7475
import java.util.Map;
7576
import java.util.Optional;
77+
import java.util.ServiceLoader;
7678
import java.util.Set;
7779
import java.util.function.Function;
7880
import java.util.function.Predicate;
@@ -83,9 +85,9 @@
8385
public class IndicesModule extends AbstractModule {
8486
private final MapperRegistry mapperRegistry;
8587

86-
public IndicesModule(List<MapperPlugin> mapperPlugins) {
87-
this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins),
88-
getFieldFilter(mapperPlugins));
88+
public IndicesModule(List<Plugin> plugins) {
89+
this.mapperRegistry = new MapperRegistry(getMappers(plugins), getMetadataMappers(plugins),
90+
getFieldFilter(plugins));
8991
}
9092

9193
public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
@@ -106,7 +108,7 @@ public static List<NamedXContentRegistry.Entry> getNamedXContents() {
106108
);
107109
}
108110

109-
public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mapperPlugins) {
111+
public static Map<String, Mapper.TypeParser> getMappers(List<Plugin> plugins) {
110112
Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
111113

112114
// builtin mappers
@@ -132,10 +134,20 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
132134
mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
133135
mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new AbstractGeometryFieldMapper.TypeParser());
134136

135-
for (MapperPlugin mapperPlugin : mapperPlugins) {
136-
for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
137-
if (mappers.put(entry.getKey(), entry.getValue()) != null) {
138-
throw new IllegalArgumentException("Mapper [" + entry.getKey() + "] is already registered");
137+
for (Plugin plugin : plugins) {
138+
for (GeoShapeFieldMapper.Extension geoExtension : ServiceLoader.load(GeoShapeFieldMapper.Extension.class,
139+
plugin.getClass().getClassLoader())) {
140+
GeoShapeFieldMapper.registerDataHandlers(geoExtension.getDataHandlerFactories());
141+
}
142+
for (AbstractGeometryFieldMapper.ParserExtension geoParserExtension :
143+
ServiceLoader.load(AbstractGeometryFieldMapper.ParserExtension.class, plugin.getClass().getClassLoader())) {
144+
GeoShapeFieldMapper.registerParserExtensions(geoParserExtension.getParserExtensions());
145+
}
146+
if (plugin instanceof MapperPlugin) {
147+
for (Map.Entry<String, Mapper.TypeParser> entry : ((MapperPlugin)plugin).getMappers().entrySet()) {
148+
if (mappers.put(entry.getKey(), entry.getValue()) != null) {
149+
throw new IllegalArgumentException("Mapper [" + entry.getKey() + "] is already registered");
150+
}
139151
}
140152
}
141153
}
@@ -165,7 +177,7 @@ private static Map<String, MetadataFieldMapper.TypeParser> initBuiltInMetadataMa
165177
return Collections.unmodifiableMap(builtInMetadataMappers);
166178
}
167179

168-
public static Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers(List<MapperPlugin> mapperPlugins) {
180+
public static Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers(List<Plugin> plugins) {
169181
Map<String, MetadataFieldMapper.TypeParser> metadataMappers = new LinkedHashMap<>();
170182

171183
int i = 0;
@@ -181,13 +193,15 @@ public static Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers(Lis
181193
}
182194
assert fieldNamesEntry != null;
183195

184-
for (MapperPlugin mapperPlugin : mapperPlugins) {
185-
for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : mapperPlugin.getMetadataMappers().entrySet()) {
186-
if (entry.getKey().equals(FieldNamesFieldMapper.NAME)) {
187-
throw new IllegalArgumentException("Plugin cannot contain metadata mapper [" + FieldNamesFieldMapper.NAME + "]");
188-
}
189-
if (metadataMappers.put(entry.getKey(), entry.getValue()) != null) {
190-
throw new IllegalArgumentException("MetadataFieldMapper [" + entry.getKey() + "] is already registered");
196+
for (Plugin plugin : plugins) {
197+
if (plugin instanceof MapperPlugin) {
198+
for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : ((MapperPlugin) plugin).getMetadataMappers().entrySet()) {
199+
if (entry.getKey().equals(FieldNamesFieldMapper.NAME)) {
200+
throw new IllegalArgumentException("Plugin cannot contain metadata mapper [" + FieldNamesFieldMapper.NAME + "]");
201+
}
202+
if (metadataMappers.put(entry.getKey(), entry.getValue()) != null) {
203+
throw new IllegalArgumentException("MetadataFieldMapper [" + entry.getKey() + "] is already registered");
204+
}
191205
}
192206
}
193207
}
@@ -204,10 +218,12 @@ public static Set<String> getBuiltInMetaDataFields() {
204218
return builtInMetadataMappers.keySet();
205219
}
206220

207-
private static Function<String, Predicate<String>> getFieldFilter(List<MapperPlugin> mapperPlugins) {
221+
private static Function<String, Predicate<String>> getFieldFilter(List<Plugin> plugins) {
208222
Function<String, Predicate<String>> fieldFilter = MapperPlugin.NOOP_FIELD_FILTER;
209-
for (MapperPlugin mapperPlugin : mapperPlugins) {
210-
fieldFilter = and(fieldFilter, mapperPlugin.getFieldFilter());
223+
for (Plugin plugin : plugins) {
224+
if (plugin instanceof MapperPlugin) {
225+
fieldFilter = and(fieldFilter, ((MapperPlugin)plugin).getFieldFilter());
226+
}
211227
}
212228
return fieldFilter;
213229
}

‎server/src/main/java/org/elasticsearch/node/Node.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,6 @@
128128
import org.elasticsearch.plugins.EnginePlugin;
129129
import org.elasticsearch.plugins.IndexStorePlugin;
130130
import org.elasticsearch.plugins.IngestPlugin;
131-
import org.elasticsearch.plugins.MapperPlugin;
132131
import org.elasticsearch.plugins.MetaDataUpgrader;
133132
import org.elasticsearch.plugins.NetworkPlugin;
134133
import org.elasticsearch.plugins.PersistentTaskPlugin;
@@ -384,7 +383,7 @@ protected Node(final Environment initialEnvironment,
384383
final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool, clusterInfoService);
385384
ClusterModule clusterModule = new ClusterModule(settings, clusterService, clusterPlugins, clusterInfoService);
386385
modules.add(clusterModule);
387-
IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
386+
IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(Plugin.class));
388387
modules.add(indicesModule);
389388

390389
SearchModule searchModule = new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class));

0 commit comments

Comments
 (0)