diff --git a/docs/changelog/108764.yaml b/docs/changelog/108764.yaml new file mode 100644 index 0000000000000..94de27eb52c9b --- /dev/null +++ b/docs/changelog/108764.yaml @@ -0,0 +1,6 @@ +pr: 108764 +summary: ST_DISTANCE Function +area: ES|QL +type: enhancement +issues: + - 108212 diff --git a/docs/changelog/109967.yaml b/docs/changelog/109967.yaml new file mode 100644 index 0000000000000..cfc6b6462954b --- /dev/null +++ b/docs/changelog/109967.yaml @@ -0,0 +1,5 @@ +pr: 109967 +summary: Default the HF service to cosine similarity +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/110035.yaml b/docs/changelog/110035.yaml new file mode 100644 index 0000000000000..670c58240d835 --- /dev/null +++ b/docs/changelog/110035.yaml @@ -0,0 +1,5 @@ +pr: 110035 +summary: Fix equals and hashcode for `SingleValueQuery.LuceneQuery` +area: ES|QL +type: bug +issues: [] diff --git a/docs/reference/esql/functions/description/st_distance.asciidoc b/docs/reference/esql/functions/description/st_distance.asciidoc new file mode 100644 index 0000000000000..b27fcef0eb4f7 --- /dev/null +++ b/docs/reference/esql/functions/description/st_distance.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the distance between two points. For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. For geographic geometries, this is the circular distance along the great circle in meters. diff --git a/docs/reference/esql/functions/examples/st_distance.asciidoc b/docs/reference/esql/functions/examples/st_distance.asciidoc new file mode 100644 index 0000000000000..60da852eff736 --- /dev/null +++ b/docs/reference/esql/functions/examples/st_distance.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/spatial.csv-spec[tag=st_distance-airports] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/spatial.csv-spec[tag=st_distance-airports-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/st_distance.json b/docs/reference/esql/functions/kibana/definition/st_distance.json new file mode 100644 index 0000000000000..448e0d54051da --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/st_distance.json @@ -0,0 +1,47 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "st_distance", + "description" : "Computes the distance between two points.\nFor cartesian geometries, this is the pythagorean distance in the same units as the original coordinates.\nFor geographic geometries, this is the circular distance along the great circle in meters.", + "signatures" : [ + { + "params" : [ + { + "name" : "geomA", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`." + }, + { + "name" : "geomB", + "type" : "cartesian_point", + "optional" : false, + "description" : "Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_point` and `cartesian_point` parameters." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "geomA", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`." + }, + { + "name" : "geomB", + "type" : "geo_point", + "optional" : false, + "description" : "Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_point` and `cartesian_point` parameters." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM airports\n| WHERE abbrev == \"CPH\"\n| EVAL distance = ST_DISTANCE(location, city_location)\n| KEEP abbrev, name, location, city_location, distance" + ] +} diff --git a/docs/reference/esql/functions/kibana/docs/st_distance.md b/docs/reference/esql/functions/kibana/docs/st_distance.md new file mode 100644 index 0000000000000..7ea2d5a255357 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/st_distance.md @@ -0,0 +1,15 @@ + + +### ST_DISTANCE +Computes the distance between two points. +For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. +For geographic geometries, this is the circular distance along the great circle in meters. + +``` +FROM airports +| WHERE abbrev == "CPH" +| EVAL distance = ST_DISTANCE(location, city_location) +| KEEP abbrev, name, location, city_location, distance +``` diff --git a/docs/reference/esql/functions/layout/st_distance.asciidoc b/docs/reference/esql/functions/layout/st_distance.asciidoc new file mode 100644 index 0000000000000..159b071ce63a7 --- /dev/null +++ b/docs/reference/esql/functions/layout/st_distance.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-st_distance]] +=== `ST_DISTANCE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/st_distance.svg[Embedded,opts=inline] + +include::../parameters/st_distance.asciidoc[] +include::../description/st_distance.asciidoc[] +include::../types/st_distance.asciidoc[] +include::../examples/st_distance.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/st_distance.asciidoc b/docs/reference/esql/functions/parameters/st_distance.asciidoc new file mode 100644 index 0000000000000..f32433dfbf6fb --- /dev/null +++ b/docs/reference/esql/functions/parameters/st_distance.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`geomA`:: +Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. + +`geomB`:: +Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_point` and `cartesian_point` parameters. diff --git a/docs/reference/esql/functions/signature/st_distance.svg b/docs/reference/esql/functions/signature/st_distance.svg new file mode 100644 index 0000000000000..1831a139a719f --- /dev/null +++ b/docs/reference/esql/functions/signature/st_distance.svg @@ -0,0 +1 @@ +ST_DISTANCE(geomA,geomB) \ No newline at end of file diff --git a/docs/reference/esql/functions/spatial-functions.asciidoc b/docs/reference/esql/functions/spatial-functions.asciidoc index d143681fcf2f2..79acc2028d983 100644 --- a/docs/reference/esql/functions/spatial-functions.asciidoc +++ b/docs/reference/esql/functions/spatial-functions.asciidoc @@ -14,6 +14,7 @@ * experimental:[] <> * experimental:[] <> * experimental:[] <> +* experimental:[] <> // end::spatial_list[] include::layout/st_intersects.asciidoc[] @@ -22,3 +23,4 @@ include::layout/st_contains.asciidoc[] include::layout/st_within.asciidoc[] include::layout/st_x.asciidoc[] include::layout/st_y.asciidoc[] +include::layout/st_distance.asciidoc[] diff --git a/docs/reference/esql/functions/types/st_distance.asciidoc b/docs/reference/esql/functions/types/st_distance.asciidoc new file mode 100644 index 0000000000000..c6ae485f3f535 --- /dev/null +++ b/docs/reference/esql/functions/types/st_distance.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +geomA | geomB | result +cartesian_point | cartesian_point | double +geo_point | geo_point | double +|=== diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 22ec4fe8fa728..1cbe97c45549e 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -257,11 +257,11 @@ It can be the ID of either a built-in model (for example, `.multilingual-e5-smal `num_allocations`::: (Required, integer) -The number of model allocations to create. `num_allocations` must not exceed the number of available processors per node divided by the `num_threads`. +The total number of allocations this model is assigned across machine learning nodes. Increasing this value generally increases the throughput. `num_threads`::: (Required, integer) -The number of threads to use by each model allocation. `num_threads` must not exceed the number of available processors per node divided by the number of allocations. +Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. Must be a power of 2. Max allowed value is 32. ===== @@ -272,11 +272,11 @@ Must be a power of 2. Max allowed value is 32. `num_allocations`::: (Required, integer) -The number of model allocations to create. `num_allocations` must not exceed the number of available processors per node divided by the `num_threads`. +The total number of allocations this model is assigned across machine learning nodes. Increasing this value generally increases the throughput. `num_threads`::: (Required, integer) -The number of threads to use by each model allocation. `num_threads` must not exceed the number of available processors per node divided by the number of allocations. +Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. Must be a power of 2. Max allowed value is 32. ===== diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 4678215dd5b60..5c1cace6119aa 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -323,7 +323,7 @@ private static void setupPrefixTrees(GeoShapeFieldType ft) { private GeoShapeFieldType buildFieldType(LegacyGeoShapeParser parser, MapperBuilderContext context) { GeoShapeFieldType ft = new GeoShapeFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get(), orientation.get().value(), parser, @@ -352,7 +352,7 @@ private static int getLevels(int treeLevels, double precisionInMeters, int defau public LegacyGeoShapeFieldMapper build(MapperBuilderContext context) { LegacyGeoShapeParser parser = new LegacyGeoShapeParser(); GeoShapeFieldType ft = buildFieldType(parser, context); - return new LegacyGeoShapeFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); + return new LegacyGeoShapeFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } @@ -610,7 +610,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { return new Builder( - simpleName(), + leafName(), indexCreatedVersion, builder.ignoreMalformed.getDefaultValue().value(), builder.coerce.getDefaultValue().value() diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index bf81003f5e1f4..fec8656dcde4b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -127,7 +127,7 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); TextSearchInfo tsi = new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchQuoteAnalyzer); MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), tsi, indexAnalyzer, context.isSourceSynthetic(), @@ -140,7 +140,15 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { public MatchOnlyTextFieldMapper build(MapperBuilderContext context) { MatchOnlyTextFieldType tft = buildFieldType(context); MultiFields multiFields = multiFieldsBuilder.build(this, context); - return new MatchOnlyTextFieldMapper(name(), Defaults.FIELD_TYPE, tft, multiFields, copyTo, context.isSourceSynthetic(), this); + return new MatchOnlyTextFieldMapper( + leafName(), + Defaults.FIELD_TYPE, + tft, + multiFields, + copyTo, + context.isSourceSynthetic(), + this + ); } } @@ -397,7 +405,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexCreatedVersion, indexAnalyzers).init(this); + return new Builder(leafName(), indexCreatedVersion, indexAnalyzers).init(this); } @Override @@ -439,7 +447,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), simpleName(), null) { + return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName(), null) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index c058dddd8f875..428729f1a20bb 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -91,9 +91,9 @@ protected Parameter[] getParameters() { @Override public RankFeatureFieldMapper build(MapperBuilderContext context) { return new RankFeatureFieldMapper( - name(), + leafName(), new RankFeatureFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), meta.getValue(), positiveScoreImpact.getValue(), nullValue.getValue() @@ -233,6 +233,6 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index 5f0d44d1fb796..f859bb68401b4 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -64,8 +64,8 @@ protected Parameter[] getParameters() { @Override public RankFeaturesFieldMapper build(MapperBuilderContext context) { return new RankFeaturesFieldMapper( - name(), - new RankFeaturesFieldType(context.buildFullName(name()), meta.getValue(), positiveScoreImpact.getValue()), + leafName(), + new RankFeaturesFieldType(context.buildFullName(leafName()), meta.getValue(), positiveScoreImpact.getValue()), multiFieldsBuilder.build(this, context), copyTo, positiveScoreImpact.getValue() @@ -137,7 +137,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index f472ce0855625..688bd267eed51 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -187,7 +187,7 @@ protected Parameter[] getParameters() { @Override public ScaledFloatFieldMapper build(MapperBuilderContext context) { ScaledFloatFieldType type = new ScaledFloatFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.getValue(), stored.getValue(), hasDocValues.getValue(), @@ -198,7 +198,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { indexMode ); return new ScaledFloatFieldMapper( - name(), + leafName(), type, multiFieldsBuilder.build(this, context), copyTo, @@ -511,7 +511,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this); } @Override @@ -729,7 +729,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed.value()) { + return new SortedNumericDocValuesSyntheticFieldLoader(name(), leafName(), ignoreMalformed.value()) { @Override protected void writeValue(XContentBuilder b, long value) throws IOException { b.value(decodeForSyntheticSource(value, scalingFactor)); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index a5e011d5772f0..d521f9b2d2a31 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -187,7 +187,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), fieldType, similarity.getValue(), analyzers.getSearchAnalyzer(), @@ -202,7 +202,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { prefixft.setIndexOptions(fieldType.indexOptions()); prefixft.setOmitNorms(true); prefixft.setStored(false); - final String fullName = context.buildFullName(name()); + final String fullName = context.buildFullName(leafName()); // wrap the root field's index analyzer with shingles and edge ngrams final Analyzer prefixIndexWrapper = SearchAsYouTypeAnalyzer.withShingleAndPrefix( indexAnalyzer.analyzer(), @@ -228,7 +228,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { final int shingleSize = i + 2; FieldType shingleft = new FieldType(fieldType); shingleft.setStored(false); - String fieldName = getShingleFieldName(context.buildFullName(name()), shingleSize); + String fieldName = getShingleFieldName(context.buildFullName(leafName()), shingleSize); // wrap the root field's index, search, and search quote analyzers with shingles final SearchAsYouTypeAnalyzer shingleIndexWrapper = SearchAsYouTypeAnalyzer.withShingle( indexAnalyzer.analyzer(), @@ -260,7 +260,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); return new SearchAsYouTypeFieldMapper( - name(), + leafName(), ft, copyTo, indexAnalyzers, @@ -721,7 +721,7 @@ protected String contentType() { } public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this); + return new Builder(leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this); } public static String getShingleFieldName(String parentField, int shingleSize) { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index 831306a8e8594..a7090cd405684 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -77,17 +77,17 @@ protected Parameter[] getParameters() { @Override public TokenCountFieldMapper build(MapperBuilderContext context) { if (analyzer.getValue() == null) { - throw new MapperParsingException("Analyzer must be set for field [" + name() + "] but wasn't."); + throw new MapperParsingException("Analyzer must be set for field [" + leafName() + "] but wasn't."); } MappedFieldType ft = new TokenCountFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), index.getValue(), store.getValue(), hasDocValues.getValue(), nullValue.getValue(), meta.getValue() ); - return new TokenCountFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, this); + return new TokenCountFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, this); } } @@ -213,6 +213,6 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index e1df6c130c9fe..3314912a5a3ba 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -123,16 +123,16 @@ public ParentJoinFieldMapper build(MapperBuilderContext context) { "Adding multifields to [" + CONTENT_TYPE + "] mappers has no effect and will be forbidden in future" ); } - checkObjectOrNested(context, name()); + checkObjectOrNested(context, leafName()); final Map parentIdFields = new HashMap<>(); relations.get() .stream() - .map(relation -> new ParentIdFieldMapper(name() + "#" + relation.parent(), eagerGlobalOrdinals.get())) + .map(relation -> new ParentIdFieldMapper(leafName() + "#" + relation.parent(), eagerGlobalOrdinals.get())) .forEach(mapper -> parentIdFields.put(mapper.name(), mapper)); - Joiner joiner = new Joiner(name(), relations.get()); + Joiner joiner = new Joiner(leafName(), relations.get()); return new ParentJoinFieldMapper( - name(), - new JoinFieldType(context.buildFullName(name()), joiner, meta.get()), + leafName(), + new JoinFieldType(context.buildFullName(leafName()), joiner, meta.get()), Collections.unmodifiableMap(parentIdFields), eagerGlobalOrdinals.get(), relations.get() @@ -249,7 +249,7 @@ protected boolean supportsParsingObject() { @Override public void parse(DocumentParserContext context) throws IOException { - context.path().add(simpleName()); + context.path().add(leafName()); XContentParser.Token token = context.parser().currentToken(); String name = null; String parent = null; @@ -329,7 +329,7 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 954dd9db0d376..f4738bf41b414 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -100,8 +100,9 @@ public class PercolatorFieldMapper extends FieldMapper { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), searchExecutionContext, mapUnmappedFieldsAsText, indexCreatedVersion, clusterTransportVersion) - .init(this); + return new Builder(leafName(), searchExecutionContext, mapUnmappedFieldsAsText, indexCreatedVersion, clusterTransportVersion).init( + this + ); } static class Builder extends FieldMapper.Builder { @@ -135,10 +136,10 @@ protected Parameter[] getParameters() { @Override public PercolatorFieldMapper build(MapperBuilderContext context) { - PercolatorFieldType fieldType = new PercolatorFieldType(context.buildFullName(name()), meta.getValue()); + PercolatorFieldType fieldType = new PercolatorFieldType(context.buildFullName(leafName()), meta.getValue()); // TODO should percolator even allow multifields? MultiFields multiFields = multiFieldsBuilder.build(this, context); - context = context.createChildContext(name(), null); + context = context.createChildContext(leafName(), null); KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder( EXTRACTED_TERMS_FIELD_NAME, context, @@ -162,7 +163,7 @@ public PercolatorFieldMapper build(MapperBuilderContext context) { fieldType.mapUnmappedFieldsAsText = mapUnmappedFieldsAsText; return new PercolatorFieldMapper( - name(), + leafName(), fieldType, multiFields, copyTo, diff --git a/muted-tests.yml b/muted-tests.yml index a17e95e9a5b3f..4c33c5fb1839a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -77,6 +77,13 @@ tests: method: "testBasicAsyncExecution" - class: "org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests" issue: "https://github.com/elastic/elasticsearch/issues/110015" +- class: "org.elasticsearch.painless.LangPainlessClientYamlTestSuiteIT" + issue: "https://github.com/elastic/elasticsearch/issues/110032" + method: "test {yaml=painless/140_dense_vector_basic/Test hamming distance fails\ + \ on float}" +- class: "org.elasticsearch.action.admin.indices.rollover.RolloverIT" + issue: "https://github.com/elastic/elasticsearch/issues/110034" + method: "testRolloverWithClosedWriteIndex" # Examples: diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 1da274ff236da..224b247f446f0 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -327,7 +327,7 @@ public ICUCollationKeywordFieldMapper build(MapperBuilderContext context) { final CollatorParams params = collatorParams(); final Collator collator = params.buildCollator(); CollationFieldType ft = new CollationFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.getValue(), stored.getValue(), hasDocValues.getValue(), @@ -337,7 +337,7 @@ public ICUCollationKeywordFieldMapper build(MapperBuilderContext context) { meta.getValue() ); return new ICUCollationKeywordFieldMapper( - name(), + leafName(), buildFieldType(), ft, multiFieldsBuilder.build(this, context), @@ -508,7 +508,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index db817917ff29e..11a899500fbbf 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -139,7 +139,7 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder wrapAnalyzer(analyzers.getSearchQuoteAnalyzer()) ); return new AnnotatedTextFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), store.getValue(), tsi, context.isSourceSynthetic(), @@ -158,12 +158,12 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { if (analyzers.positionIncrementGap.isConfigured()) { if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException( - "Cannot set position_increment_gap on field [" + name() + "] without positions enabled" + "Cannot set position_increment_gap on field [" + leafName() + "] without positions enabled" ); } } return new AnnotatedTextFieldMapper( - name(), + leafName(), fieldType, buildFieldType(fieldType, context, multiFields), multiFields, @@ -564,7 +564,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { return new Builder( - simpleName(), + leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers, builder.isSyntheticSourceEnabledViaIndexMode @@ -584,7 +584,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { ); } if (fieldType.stored()) { - return new StringStoredFieldFieldLoader(name(), simpleName(), null) { + return new StringStoredFieldFieldLoader(name(), leafName(), null) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); @@ -594,7 +594,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return kwd.syntheticFieldLoader(simpleName()); + return kwd.syntheticFieldLoader(leafName()); } throw new IllegalArgumentException( diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 44f52105f64c9..27ca049297c79 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -55,8 +55,8 @@ protected Parameter[] getParameters() { @Override public Murmur3FieldMapper build(MapperBuilderContext context) { return new Murmur3FieldMapper( - name(), - new Murmur3FieldType(context.buildFullName(name()), stored.getValue(), meta.getValue()), + leafName(), + new Murmur3FieldType(context.buildFullName(leafName()), stored.getValue(), meta.getValue()), multiFieldsBuilder.build(this, context), copyTo ); @@ -100,7 +100,7 @@ protected Murmur3FieldMapper(String simpleName, MappedFieldType mappedFieldType, @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 9d3ce1c99b553..fcd91bbb193d1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -859,7 +859,7 @@ protected String contentType() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new StringStoredFieldFieldLoader(name(), simpleName(), null) { + return new StringStoredFieldFieldLoader(name(), leafName(), null) { @Override protected void write(XContentBuilder b, Object value) throws IOException { BytesRef ref = (BytesRef) value; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 0f9c77e810924..d62ab1013dcff 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -197,6 +197,8 @@ static TransportVersion def(int id) { public static final TransportVersion ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD = def(8_688_00_0); public static final TransportVersion FAILURE_STORE_LAZY_CREATION = def(8_689_00_0); public static final TransportVersion SNAPSHOT_REQUEST_TIMEOUTS = def(8_690_00_0); + public static final TransportVersion INDEX_METADATA_MAPPINGS_UPDATED_VERSION = def(8_691_00_0); + public static final TransportVersion ML_INFERENCE_ELAND_SETTINGS_ADDED = def(8_692_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 64809c963cb6d..2c14226f9eb96 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -535,6 +535,7 @@ public Iterator> settings() { static final String KEY_MAPPINGS_HASH = "mappings_hash"; static final String KEY_ALIASES = "aliases"; static final String KEY_ROLLOVER_INFOS = "rollover_info"; + static final String KEY_MAPPINGS_UPDATED_VERSION = "mappings_updated_version"; static final String KEY_SYSTEM = "system"; static final String KEY_TIMESTAMP_RANGE = "timestamp_range"; public static final String KEY_PRIMARY_TERMS = "primary_terms"; @@ -594,6 +595,7 @@ public Iterator> settings() { private final DiscoveryNodeFilters initialRecoveryFilters; private final IndexVersion indexCreatedVersion; + private final IndexVersion mappingsUpdatedVersion; private final IndexVersion indexCompatibilityVersion; private final ActiveShardCount waitForActiveShards; @@ -659,6 +661,7 @@ private IndexMetadata( final DiscoveryNodeFilters includeFilters, final DiscoveryNodeFilters excludeFilters, final IndexVersion indexCreatedVersion, + final IndexVersion mappingsUpdatedVersion, final int routingNumShards, final int routingPartitionSize, final List routingPaths, @@ -689,6 +692,7 @@ private IndexMetadata( this.version = version; assert mappingVersion >= 0 : mappingVersion; this.mappingVersion = mappingVersion; + this.mappingsUpdatedVersion = mappingsUpdatedVersion; assert settingsVersion >= 0 : settingsVersion; this.settingsVersion = settingsVersion; assert aliasesVersion >= 0 : aliasesVersion; @@ -767,6 +771,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.includeFilters, this.excludeFilters, this.indexCreatedVersion, + this.mappingsUpdatedVersion, this.routingNumShards, this.routingPartitionSize, this.routingPaths, @@ -826,6 +831,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.includeFilters, this.excludeFilters, this.indexCreatedVersion, + this.mappingsUpdatedVersion, this.routingNumShards, this.routingPartitionSize, this.routingPaths, @@ -883,6 +889,7 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) { this.includeFilters, this.excludeFilters, this.indexCreatedVersion, + this.mappingsUpdatedVersion, this.routingNumShards, this.routingPartitionSize, this.routingPaths, @@ -940,6 +947,7 @@ public IndexMetadata withTimestampRange(IndexLongFieldRange timestampRange) { this.includeFilters, this.excludeFilters, this.indexCreatedVersion, + this.mappingsUpdatedVersion, this.routingNumShards, this.routingPartitionSize, this.routingPaths, @@ -993,6 +1001,7 @@ public IndexMetadata withIncrementedVersion() { this.includeFilters, this.excludeFilters, this.indexCreatedVersion, + this.mappingsUpdatedVersion, this.routingNumShards, this.routingPartitionSize, this.routingPaths, @@ -1037,6 +1046,10 @@ public long getMappingVersion() { return mappingVersion; } + public IndexVersion getMappingsUpdatedVersion() { + return mappingsUpdatedVersion; + } + public long getSettingsVersion() { return settingsVersion; } @@ -1497,6 +1510,7 @@ private static class IndexMetadataDiff implements Diff { private final Diff> customData; private final Diff>> inSyncAllocationIds; private final Diff> rolloverInfos; + private final IndexVersion mappingsUpdatedVersion; private final boolean isSystem; private final IndexLongFieldRange timestampRange; private final IndexMetadataStats stats; @@ -1534,6 +1548,7 @@ private static class IndexMetadataDiff implements Diff { DiffableUtils.StringSetValueSerializer.getInstance() ); rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer()); + mappingsUpdatedVersion = after.mappingsUpdatedVersion; isSystem = after.isSystem; timestampRange = after.timestampRange; stats = after.stats; @@ -1594,6 +1609,11 @@ private static class IndexMetadataDiff implements Diff { DiffableUtils.getStringKeySerializer(), ROLLOVER_INFO_DIFF_VALUE_READER ); + if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) { + mappingsUpdatedVersion = IndexVersion.readVersion(in); + } else { + mappingsUpdatedVersion = IndexVersions.ZERO; + } if (in.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { isSystem = in.readBoolean(); } else { @@ -1638,6 +1658,9 @@ public void writeTo(StreamOutput out) throws IOException { customData.writeTo(out); inSyncAllocationIds.writeTo(out); rolloverInfos.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) { + IndexVersion.writeVersion(mappingsUpdatedVersion, out); + } if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { out.writeBoolean(isSystem); } @@ -1667,6 +1690,7 @@ public IndexMetadata apply(IndexMetadata part) { builder.mapping = mappings.apply( ImmutableOpenMap.builder(1).fPut(MapperService.SINGLE_MAPPING_NAME, part.mapping).build() ).get(MapperService.SINGLE_MAPPING_NAME); + builder.mappingsUpdatedVersion = mappingsUpdatedVersion; builder.inferenceFields.putAllFromMap(inferenceFields.apply(part.inferenceFields)); builder.aliases.putAllFromMap(aliases.apply(part.aliases)); builder.customMetadata.putAllFromMap(customData.apply(part.customData)); @@ -1738,6 +1762,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function DiffableUtils.StringSetValueSerializer.getInstance().write(v, o) ); out.writeCollection(rolloverInfos.values()); + if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_METADATA_MAPPINGS_UPDATED_VERSION)) { + IndexVersion.writeVersion(mappingsUpdatedVersion, out); + } if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { out.writeBoolean(isSystem); } @@ -1835,6 +1865,7 @@ public static class Builder { private long[] primaryTerms = null; private Settings settings = Settings.EMPTY; private MappingMetadata mapping; + private IndexVersion mappingsUpdatedVersion = IndexVersion.current(); private final ImmutableOpenMap.Builder inferenceFields; private final ImmutableOpenMap.Builder aliases; private final ImmutableOpenMap.Builder customMetadata; @@ -1873,6 +1904,7 @@ public Builder(IndexMetadata indexMetadata) { this.customMetadata = ImmutableOpenMap.builder(indexMetadata.customData); this.routingNumShards = indexMetadata.routingNumShards; this.inSyncAllocationIds = new HashMap<>(indexMetadata.inSyncAllocationIds); + this.mappingsUpdatedVersion = indexMetadata.mappingsUpdatedVersion; this.rolloverInfos = ImmutableOpenMap.builder(indexMetadata.rolloverInfos); this.isSystem = indexMetadata.isSystem; this.timestampRange = indexMetadata.timestampRange; @@ -1964,6 +1996,11 @@ public Builder putMapping(MappingMetadata mappingMd) { return this; } + public Builder mappingsUpdatedVersion(IndexVersion indexVersion) { + this.mappingsUpdatedVersion = indexVersion; + return this; + } + public Builder state(State state) { this.state = state; return this; @@ -2291,6 +2328,7 @@ IndexMetadata build(boolean repair) { includeFilters, excludeFilters, indexCreatedVersion, + mappingsUpdatedVersion, getRoutingNumShards(), routingPartitionSize, routingPaths, @@ -2421,6 +2459,8 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build rolloverInfo.toXContent(builder, params); } builder.endObject(); + + builder.field(KEY_MAPPINGS_UPDATED_VERSION, indexMetadata.mappingsUpdatedVersion); builder.field(KEY_SYSTEM, indexMetadata.isSystem); builder.startObject(KEY_TIMESTAMP_RANGE); @@ -2589,6 +2629,7 @@ public static IndexMetadata fromXContent(XContentParser parser, Map builder.mappingsUpdatedVersion(IndexVersion.fromId(parser.intValue())); case KEY_WRITE_LOAD_FORECAST -> builder.indexWriteLoadForecast(parser.doubleValue()); case KEY_SHARD_SIZE_FORECAST -> builder.shardSizeInBytesForecast(parser.longValue()); default -> throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]"); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 4e714b96f64c7..4ed18489c44b0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -27,6 +27,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -199,7 +200,8 @@ private static ClusterState applyRequest( indexMetadataBuilder.putInferenceFields(docMapper.mappers().inferenceFields()); } if (updatedMapping) { - indexMetadataBuilder.mappingVersion(1 + indexMetadataBuilder.mappingVersion()); + indexMetadataBuilder.mappingVersion(1 + indexMetadataBuilder.mappingVersion()) + .mappingsUpdatedVersion(IndexVersion.current()); } /* * This implicitly increments the index metadata version and builds the index metadata. This means that we need to have diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 9dbbdd597a4ce..13c2fabd6b3df 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; @@ -246,6 +247,7 @@ static void prepareBackingIndex( imb.settings(settingsUpdate.build()) .settingsVersion(im.getSettingsVersion() + 1) .mappingVersion(im.getMappingVersion() + 1) + .mappingsUpdatedVersion(IndexVersion.current()) .putMapping(new MappingMetadata(mapper)); b.put(imb); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 5552f62bf8ce4..3ba83ffe7086f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -70,8 +70,8 @@ public BinaryFieldMapper.Builder docValues(boolean hasDocValues) { @Override public BinaryFieldMapper build(MapperBuilderContext context) { return new BinaryFieldMapper( - name(), - new BinaryFieldType(context.buildFullName(name()), stored.getValue(), hasDocValues.getValue(), meta.getValue()), + leafName(), + new BinaryFieldType(context.buildFullName(leafName()), stored.getValue(), hasDocValues.getValue(), meta.getValue()), multiFieldsBuilder.build(this, context), copyTo, this @@ -192,7 +192,7 @@ public void indexValue(DocumentParserContext context, byte[] value) { @Override public FieldMapper.Builder getMergeBuilder() { - return new BinaryFieldMapper.Builder(simpleName(), isSyntheticSourceEnabledViaIndexMode).init(this); + return new BinaryFieldMapper.Builder(leafName(), isSyntheticSourceEnabledViaIndexMode).init(this); } @Override @@ -229,10 +229,10 @@ protected void writeValue(XContentBuilder b, BytesRef value) throws IOException case 0: return; case 1: - b.field(simpleName()); + b.field(leafName()); break; default: - b.startArray(simpleName()); + b.startArray(leafName()); } for (int i = 0; i < count; i++) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index c6b428458d2b9..aa60e78ee382e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -116,7 +116,7 @@ protected Parameter[] getParameters() { @Override public BooleanFieldMapper build(MapperBuilderContext context) { MappedFieldType ft = new BooleanFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false, stored.getValue(), docValues.getValue(), @@ -124,7 +124,14 @@ public BooleanFieldMapper build(MapperBuilderContext context) { scriptValues(), meta.getValue() ); - return new BooleanFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); + return new BooleanFieldMapper( + leafName(), + ft, + multiFieldsBuilder.build(this, context), + copyTo, + context.isSourceSynthetic(), + this + ); } private FieldValues scriptValues() { @@ -134,7 +141,7 @@ private FieldValues scriptValues() { BooleanFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), BooleanFieldScript.CONTEXT); return scriptFactory == null ? null - : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(name(), script.get().getParams(), lookup, OnScriptError.FAIL) + : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) .newInstance(ctx) .runForDoc(doc, consumer); } @@ -473,7 +480,7 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this); + return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this); } @Override @@ -506,7 +513,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed.value()) { + return new SortedNumericDocValuesSyntheticFieldLoader(name(), leafName(), ignoreMalformed.value()) { @Override protected void writeValue(XContentBuilder b, long value) throws IOException { b.value(value == 1); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 5d5ef076852a8..23272fbd354f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -82,7 +82,7 @@ public class CompletionFieldMapper extends FieldMapper { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), builder.defaultAnalyzer, builder.indexVersionCreated).init(this); + return new Builder(leafName(), builder.defaultAnalyzer, builder.indexVersionCreated).init(this); } public static class Defaults { @@ -205,9 +205,9 @@ public CompletionFieldMapper build(MapperBuilderContext context) { new CompletionAnalyzer(this.searchAnalyzer.getValue(), preserveSeparators.getValue(), preservePosInc.getValue()) ); - CompletionFieldType ft = new CompletionFieldType(context.buildFullName(name()), completionAnalyzer, meta.getValue()); + CompletionFieldType ft = new CompletionFieldType(context.buildFullName(leafName()), completionAnalyzer, meta.getValue()); ft.setContextMappings(contexts.getValue()); - return new CompletionFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, this); + return new CompletionFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, this); } private void checkCompletionContextsLimit() { @@ -224,7 +224,7 @@ private void checkCompletionContextsLimit() { + COMPLETION_CONTEXTS_LIMIT + "] completion contexts" + " in the mapping for field [" - + name() + + leafName() + "]. " + "The maximum allowed number of completion contexts in a mapping will be limited to " + "[" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index c817bed6e503e..b367d203d55b1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -293,7 +293,7 @@ DateFormatter buildFormatter() { logger.warn(() -> "Error parsing format [" + format.getValue() + "] of legacy index, falling back to default", e); return DateFormatter.forPattern(format.getDefaultValue()).withLocale(locale.getValue()); } else { - throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e); + throw new IllegalArgumentException("Error parsing [format] on field [" + leafName() + "]: " + e.getMessage(), e); } } } @@ -306,7 +306,7 @@ private FieldValues scriptValues() { return factory == null ? null : (lookup, ctx, doc, consumer) -> factory.newFactory( - name(), + leafName(), script.get().getParams(), lookup, buildFormatter(), @@ -327,7 +327,7 @@ private Long parseNullValue(DateFieldType fieldType) { return fieldType.parse(nullValue.getValue()); } catch (Exception e) { if (indexCreatedVersion.onOrAfter(IndexVersions.V_8_0_0)) { - throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e); + throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e); } else { DEPRECATION_LOGGER.warn( DeprecationCategory.MAPPINGS, @@ -335,7 +335,7 @@ private Long parseNullValue(DateFieldType fieldType) { "Error parsing [" + nullValue.getValue() + "] as date in [null_value] on field [" - + name() + + leafName() + "]); [null_value] will be ignored" ); return null; @@ -346,7 +346,7 @@ private Long parseNullValue(DateFieldType fieldType) { @Override public DateFieldMapper build(MapperBuilderContext context) { DateFieldType ft = new DateFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), index.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false, index.getValue(), store.getValue(), @@ -359,13 +359,13 @@ public DateFieldMapper build(MapperBuilderContext context) { ); Long nullTimestamp = parseNullValue(ft); - if (name().equals(DataStreamTimestampFieldMapper.DEFAULT_PATH) + if (leafName().equals(DataStreamTimestampFieldMapper.DEFAULT_PATH) && context.isDataStream() && ignoreMalformed.isConfigured() == false) { ignoreMalformed.setValue(false); } return new DateFieldMapper( - name(), + leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, @@ -898,7 +898,7 @@ private DateFieldMapper( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this); + return new Builder(leafName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this); } @Override @@ -997,7 +997,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed) { + return new SortedNumericDocValuesSyntheticFieldLoader(name(), leafName(), ignoreMalformed) { @Override protected void writeValue(XContentBuilder b, long value) throws IOException { b.value(fieldType().format(value, fieldType().dateTimeFormatter())); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 0966698277723..edbcfd51836fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -295,7 +295,7 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio context.addIgnoredField( new IgnoredSourceFieldMapper.NameValue( context.parent().name(), - context.parent().fullPath().indexOf(context.parent().simpleName()), + context.parent().fullPath().indexOf(context.parent().leafName()), XContentDataHelper.encodeXContentBuilder(tuple.v2()), context.doc() ) @@ -429,7 +429,7 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr } else if (mapper instanceof FieldMapper fieldMapper) { if (shouldFlattenObject(context, fieldMapper)) { // we pass the mapper's simpleName as parentName to the new DocumentParserContext - String currentFieldName = fieldMapper.simpleName(); + String currentFieldName = fieldMapper.leafName(); context.path().remove(); parseObjectOrNested(context.createFlattenContext(currentFieldName)); context.path().add(currentFieldName); @@ -562,7 +562,7 @@ private static void parseObjectDynamic(DocumentParserContext context, String cur throw new DocumentParsingException( context.parser().getTokenLocation(), "Tried to add nested object [" - + dynamicObjectMapper.simpleName() + + dynamicObjectMapper.leafName() + "] to object [" + context.parent().name() + "] which does not support subobjects" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java index 8aa29e6317d51..5f7220dde449e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java @@ -71,7 +71,7 @@ public Iterator iterator() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject(simpleName()).field("type", CONTENT_TYPE).field(Names.PATH, path).endObject(); + return builder.startObject(leafName()).field("type", CONTENT_TYPE).field(Names.PATH, path).endObject(); } @Override @@ -151,8 +151,8 @@ public Builder path(String path) { @Override public FieldAliasMapper build(MapperBuilderContext context) { - String fullName = context.buildFullName(name()); - return new FieldAliasMapper(name(), fullName, path); + String fullName = context.buildFullName(leafName()); + return new FieldAliasMapper(leafName(), fullName, path); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 4338a62d79ab9..04bc8a2b7b73f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -194,7 +194,7 @@ public void parse(DocumentParserContext context) throws IOException { } private void doParseMultiFields(DocumentParserContext context) throws IOException { - context.path().add(simpleName()); + context.path().add(leafName()); for (FieldMapper mapper : multiFields.mappers) { mapper.parse(context); } @@ -416,7 +416,7 @@ protected void checkIncomingMergeType(FieldMapper mergeWith) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(simpleName()); + builder.startObject(leafName()); doXContentBody(builder, params); return builder.endObject(); } @@ -514,7 +514,7 @@ public static class Builder { private boolean hasSyntheticSourceCompatibleKeywordField; public Builder add(FieldMapper.Builder builder) { - mapperBuilders.put(builder.name(), builder::build); + mapperBuilders.put(builder.leafName(), builder::build); if (builder instanceof KeywordFieldMapper.Builder kwd) { if (kwd.hasNormalizer() == false && (kwd.hasDocValues() || kwd.isStored())) { @@ -526,7 +526,7 @@ public Builder add(FieldMapper.Builder builder) { } private void add(FieldMapper mapper) { - mapperBuilders.put(mapper.simpleName(), context -> mapper); + mapperBuilders.put(mapper.leafName(), context -> mapper); if (mapper instanceof KeywordFieldMapper kwd) { if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) { @@ -536,12 +536,12 @@ private void add(FieldMapper mapper) { } private void update(FieldMapper toMerge, MapperMergeContext context) { - if (mapperBuilders.containsKey(toMerge.simpleName()) == false) { + if (mapperBuilders.containsKey(toMerge.leafName()) == false) { if (context.decrementFieldBudgetIfPossible(toMerge.getTotalFieldsCount())) { add(toMerge); } } else { - FieldMapper existing = mapperBuilders.get(toMerge.simpleName()).apply(context.getMapperBuilderContext()); + FieldMapper existing = mapperBuilders.get(toMerge.leafName()).apply(context.getMapperBuilderContext()); add(existing.merge(toMerge, context)); } } @@ -559,7 +559,7 @@ public MultiFields build(Mapper.Builder mainFieldBuilder, MapperBuilderContext c return empty(); } else { FieldMapper[] mappers = new FieldMapper[mapperBuilders.size()]; - context = context.createChildContext(mainFieldBuilder.name(), null); + context = context.createChildContext(mainFieldBuilder.leafName(), null); int i = 0; for (Map.Entry> entry : this.mapperBuilders.entrySet()) { mappers[i++] = entry.getValue().apply(context); @@ -584,7 +584,7 @@ public void parse(FieldMapper mainField, DocumentParserContext context, Supplier if (mappers.length == 0) { return; } - context.path().add(mainField.simpleName()); + context.path().add(mainField.leafName()); for (FieldMapper mapper : mappers) { mapper.parse(multiFieldContextSupplier.get()); } @@ -1314,7 +1314,7 @@ protected void merge(FieldMapper in, Conflicts conflicts, MapperMergeContext map for (Parameter param : getParameters()) { param.merge(in, conflicts); } - MapperMergeContext childContext = mapperMergeContext.createChildContext(in.simpleName(), null); + MapperMergeContext childContext = mapperMergeContext.createChildContext(in.leafName(), null); for (FieldMapper newSubField : in.multiFields.mappers) { multiFieldsBuilder.update(newSubField, childContext); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 7070c387fbb97..98f8867f386a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -99,7 +99,7 @@ final class FieldTypeLookup { for (PassThroughObjectMapper passThroughMapper : passThroughMappers) { for (Mapper subfield : passThroughMapper.mappers.values()) { if (subfield instanceof FieldMapper fieldMapper) { - String name = fieldMapper.simpleName(); + String name = fieldMapper.leafName(); // Check for conflict between PassThroughObjectMapper subfields. PassThroughObjectMapper conflict = passThroughFieldAliases.put(name, passThroughMapper); if (conflict != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index b31a61d50ecdb..36c18c7d3bbf2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -188,7 +188,7 @@ private FieldValues scriptValues() { GeoPointFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), GeoPointFieldScript.CONTEXT); return factory == null ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory(name(), script.get().getParams(), lookup, OnScriptError.FAIL) + : (lookup, ctx, doc, consumer) -> factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) .newInstance(ctx) .runForDoc(doc, consumer); } @@ -197,7 +197,7 @@ private FieldValues scriptValues() { public FieldMapper build(MapperBuilderContext context) { boolean ignoreMalformedEnabled = ignoreMalformed.get().value(); Parser geoParser = new GeoPointParser( - name(), + leafName(), (parser) -> GeoUtils.parseGeoPoint(parser, ignoreZValue.get().value()), nullValue.get(), ignoreZValue.get().value(), @@ -206,7 +206,7 @@ public FieldMapper build(MapperBuilderContext context) { context.isSourceSynthetic() && ignoreMalformedEnabled ); GeoPointFieldType ft = new GeoPointFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get() && indexCreatedVersion.isLegacyIndexVersion() == false, stored.get(), hasDocValues.get(), @@ -218,9 +218,9 @@ public FieldMapper build(MapperBuilderContext context) { indexMode ); if (this.script.get() == null) { - return new GeoPointFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, geoParser, this); + return new GeoPointFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, geoParser, this); } - return new GeoPointFieldMapper(name(), ft, geoParser, this); + return new GeoPointFieldMapper(leafName(), ft, geoParser, this); } } @@ -284,7 +284,7 @@ public GeoPointFieldMapper(String simpleName, MappedFieldType mappedFieldType, P @Override public FieldMapper.Builder getMergeBuilder() { return new Builder( - simpleName(), + leafName(), builder.scriptCompiler, builder.ignoreMalformed.getDefaultValue().value(), indexCreatedVersion, @@ -636,7 +636,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) { + return new SortedNumericDocValuesSyntheticFieldLoader(name(), leafName(), ignoreMalformed()) { final GeoPoint point = new GeoPoint(); @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 541538f65a550..32d734a622eed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -99,18 +99,18 @@ public GeoShapeFieldMapper build(MapperBuilderContext context) { ); GeoShapeParser geoShapeParser = new GeoShapeParser(geometryParser, orientation.get().value()); GeoShapeFieldType ft = new GeoShapeFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get(), orientation.get().value(), geoShapeParser, meta.get() ); return new GeoShapeFieldMapper( - name(), + leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, - new GeoShapeIndexer(orientation.get().value(), context.buildFullName(name())), + new GeoShapeIndexer(orientation.get().value(), context.buildFullName(leafName())), geoShapeParser, this ); @@ -184,7 +184,7 @@ public GeoShapeFieldMapper( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), builder.ignoreMalformed.getDefaultValue().value(), builder.coerce.getDefaultValue().value()).init( + return new Builder(leafName(), builder.ignoreMalformed.getDefaultValue().value(), builder.coerce.getDefaultValue().value()).init( this ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 1d73e256bd2e9..d8bd9d7cc55ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -131,7 +131,7 @@ private InetAddress parseNullValue() { return InetAddresses.forString(nullValueAsString); } catch (Exception e) { if (indexCreatedVersion.onOrAfter(IndexVersions.V_8_0_0)) { - throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e); + throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e); } else { DEPRECATION_LOGGER.warn( DeprecationCategory.MAPPINGS, @@ -139,7 +139,7 @@ private InetAddress parseNullValue() { "Error parsing [" + nullValue.getValue() + "] as IP in [null_value] on field [" - + name() + + leafName() + "]); [null_value] will be ignored" ); return null; @@ -154,7 +154,7 @@ private FieldValues scriptValues() { IpFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), IpFieldScript.CONTEXT); return factory == null ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory(name(), script.get().getParams(), lookup, OnScriptError.FAIL) + : (lookup, ctx, doc, consumer) -> factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) .newInstance(ctx) .runForDoc(doc, consumer); } @@ -170,9 +170,9 @@ public IpFieldMapper build(MapperBuilderContext context) { dimension.setValue(true); } return new IpFieldMapper( - name(), + leafName(), new IpFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false, stored.getValue(), hasDocValues.getValue(), @@ -589,7 +589,7 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(dimension).init(this); + return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(dimension).init(this); } @Override @@ -621,7 +621,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName(), null, ignoreMalformed) { + return new SortedSetDocValuesSyntheticFieldLoader(name(), leafName(), null, ignoreMalformed) { @Override protected BytesRef convert(BytesRef value) { byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 438964cf0a092..fca898c3914fe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -271,7 +271,7 @@ private FieldValues scriptValues() { StringFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), StringFieldScript.CONTEXT); return scriptFactory == null ? null - : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(name(), script.get().getParams(), lookup, OnScriptError.FAIL) + : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) .newInstance(ctx) .runForDoc(doc, consumer); } @@ -311,7 +311,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType ); normalizer = Lucene.KEYWORD_ANALYZER; } else { - throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name() + "]"); + throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + leafName() + "]"); } } searchAnalyzer = quoteAnalyzer = normalizer; @@ -325,7 +325,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType dimension(true); } return new KeywordFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), fieldType, normalizer, searchAnalyzer, @@ -347,7 +347,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { fieldtype = Defaults.FIELD_TYPE; } return new KeywordFieldMapper( - name(), + leafName(), fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), @@ -995,8 +995,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexAnalyzers, scriptCompiler, indexCreatedVersion).dimension(fieldType().isDimension()) - .init(this); + return new Builder(leafName(), indexAnalyzers, scriptCompiler, indexCreatedVersion).dimension(fieldType().isDimension()).init(this); } @Override @@ -1028,7 +1027,7 @@ protected SyntheticSourceMode syntheticSourceMode() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return syntheticFieldLoader(simpleName()); + return syntheticFieldLoader(leafName()); } public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String simpleName) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 7c047125a80d3..765b2d7c48fe7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -24,22 +24,21 @@ public abstract class Mapper implements ToXContentFragment, Iterable { public abstract static class Builder { - private String name; + private String leafName; - protected Builder(String name) { - setName(name); + protected Builder(String leafName) { + setLeafName(leafName); } - // TODO rename this to leafName? - public final String name() { - return this.name; + public final String leafName() { + return this.leafName; } /** Returns a newly built mapper. */ public abstract Mapper build(MapperBuilderContext context); - void setName(String name) { - this.name = internFieldName(name); + void setLeafName(String leafName) { + this.leafName = internFieldName(leafName); } } @@ -54,18 +53,15 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) { } } - private final String simpleName; + private final String leafName; - public Mapper(String simpleName) { - Objects.requireNonNull(simpleName); - this.simpleName = internFieldName(simpleName); + public Mapper(String leafName) { + Objects.requireNonNull(leafName); + this.leafName = internFieldName(leafName); } - /** Returns the simple name, which identifies this mapper against other mappers at the same level in the mappers hierarchy - * TODO: make this protected once Mapper and FieldMapper are merged together */ - // TODO rename this to leafName? - public final String simpleName() { - return simpleName; + public final String leafName() { + return leafName; } /** Returns the canonical name which uniquely identifies the mapper against other mappers in a type. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 83e6984285749..77eac1d50f707 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -226,7 +226,7 @@ private static boolean assertMapperNamesInterned(Map mappers, Ma private static void assertNamesInterned(String name, Mapper mapper) { assert name == name.intern(); assert mapper.name() == mapper.name().intern(); - assert mapper.simpleName() == mapper.simpleName().intern(); + assert mapper.leafName() == mapper.leafName().intern(); if (mapper instanceof ObjectMapper) { ((ObjectMapper) mapper).mappers.forEach(MappingLookup::assertNamesInterned); } @@ -362,7 +362,7 @@ private void checkFieldNameLengthLimit(long limit) { private static void validateMapperNameIn(Collection mappers, long limit) { for (Mapper mapper : mappers) { - String name = mapper.simpleName(); + String name = mapper.leafName(); if (name.length() > limit) { throw new IllegalArgumentException("Field name [" + name + "] is longer than the limit of [" + limit + "] characters"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 93ffbbf552071..433d9a1c5b653 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -188,7 +188,7 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params) if (mergeBuilder == null || mergeBuilder.isConfigured() == false) { return builder; } - builder.startObject(simpleName()); + builder.startObject(leafName()); getMergeBuilder().toXContent(builder, params); return builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index 4bc633296a832..21ddf0fc1c115 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -82,7 +82,7 @@ public NestedObjectMapper build(MapperBuilderContext context) { } parentTypeFilter = Queries.newNonNestedFilter(indexCreatedVersion); } - final String fullPath = context.buildFullName(name()); + final String fullPath = context.buildFullName(leafName()); final String nestedTypePath; if (indexCreatedVersion.before(IndexVersions.V_8_0_0)) { nestedTypePath = "__" + fullPath; @@ -91,14 +91,14 @@ public NestedObjectMapper build(MapperBuilderContext context) { } final Query nestedTypeFilter = NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath); NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext( - context.buildFullName(name()), + context.buildFullName(leafName()), nestedTypeFilter, parentIncludedInRoot, context.getDynamic(dynamic), context.getMergeReason() ); return new NestedObjectMapper( - name(), + leafName(), fullPath, buildMappers(nestedContext), enabled, @@ -244,7 +244,7 @@ public Map getChildren() { @Override public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(simpleName(), indexVersionCreated, bitsetProducer); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer); builder.enabled = enabled; builder.dynamic = dynamic; builder.includeInRoot = includeInRoot; @@ -255,7 +255,7 @@ public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { @Override NestedObjectMapper withoutMappers() { return new NestedObjectMapper( - simpleName(), + leafName(), fullPath(), Map.of(), enabled, @@ -272,7 +272,7 @@ NestedObjectMapper withoutMappers() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(simpleName()); + builder.startObject(leafName()); builder.field("type", CONTENT_TYPE); if (includeInParent.explicit() && includeInParent.value()) { builder.field("include_in_parent", includeInParent.value()); @@ -330,7 +330,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex } } return new NestedObjectMapper( - simpleName(), + leafName(), fullPath(), mergeResult.mappers(), mergeResult.enabled(), @@ -451,12 +451,12 @@ public boolean hasValue() { public void write(XContentBuilder b) throws IOException { assert (children != null && children.size() > 0); if (children.size() == 1) { - b.startObject(simpleName()); + b.startObject(leafName()); leafStoredFieldLoader.advanceTo(children.get(0)); leafSourceLoader.write(leafStoredFieldLoader, children.get(0), b); b.endObject(); } else { - b.startArray(simpleName()); + b.startArray(leafName()); for (int childId : children) { b.startObject(); leafStoredFieldLoader.advanceTo(childId); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index fce0fb7a83ae4..979f378e6a3bf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -221,7 +221,7 @@ private FieldValues scriptValues() { if (this.script.get() == null) { return null; } - return type.compile(name(), script.get(), scriptCompiler); + return type.compile(leafName(), script.get(), scriptCompiler); } public Builder dimension(boolean dimension) { @@ -265,8 +265,15 @@ public NumberFieldMapper build(MapperBuilderContext context) { dimension.setValue(true); } - MappedFieldType ft = new NumberFieldType(context.buildFullName(name()), this); - return new NumberFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); + MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this); + return new NumberFieldMapper( + leafName(), + ft, + multiFieldsBuilder.build(this, context), + copyTo, + context.isSourceSynthetic(), + this + ); } } @@ -1963,7 +1970,7 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion, indexMode) + return new Builder(leafName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion, indexMode) .dimension(dimension) .metric(metricType) .allowMultipleValues(allowMultipleValues) @@ -1999,7 +2006,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return type.syntheticFieldLoader(name(), simpleName(), ignoreMalformed.value()); + return type.syntheticFieldLoader(name(), leafName(), ignoreMalformed.value()); } // For testing only: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 356c103756bac..f2a81f05fe0fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -166,7 +166,7 @@ protected final Map buildMappers(MapperBuilderContext mapperBuil Map mappers = new HashMap<>(); for (Mapper.Builder builder : mappersBuilders) { Mapper mapper = builder.build(mapperBuilderContext); - Mapper existing = mappers.get(mapper.simpleName()); + Mapper existing = mappers.get(mapper.leafName()); if (existing != null) { // The same mappings or document may hold the same field twice, either because duplicated JSON keys are allowed or // the same field is provided using the object notation as well as the dot notation at the same time. @@ -177,9 +177,9 @@ protected final Map buildMappers(MapperBuilderContext mapperBuil } if (subobjects.value() == false && mapper instanceof ObjectMapper objectMapper) { // We're parsing a mapping that has set `subobjects: false` but has defined sub-objects - objectMapper.asFlattenedFieldMappers(mapperBuilderContext).forEach(m -> mappers.put(m.simpleName(), m)); + objectMapper.asFlattenedFieldMappers(mapperBuilderContext).forEach(m -> mappers.put(m.leafName(), m)); } else { - mappers.put(mapper.simpleName(), mapper); + mappers.put(mapper.leafName(), mapper); } } return mappers; @@ -188,13 +188,13 @@ protected final Map buildMappers(MapperBuilderContext mapperBuil @Override public ObjectMapper build(MapperBuilderContext context) { return new ObjectMapper( - name(), - context.buildFullName(name()), + leafName(), + context.buildFullName(leafName()), enabled, subobjects, storeArraySource, dynamic, - buildMappers(context.createChildContext(name(), dynamic)) + buildMappers(context.createChildContext(leafName(), dynamic)) ); } } @@ -325,7 +325,7 @@ protected static void parseProperties( "Tried to add nested object [" + fieldName + "] to object [" - + objBuilder.name() + + objBuilder.leafName() + "] which does not support subobjects" ); } @@ -418,7 +418,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate * @return a Builder that will produce an empty ObjectMapper with the same configuration as this one */ public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - ObjectMapper.Builder builder = new ObjectMapper.Builder(simpleName(), subobjects); + ObjectMapper.Builder builder = new ObjectMapper.Builder(leafName(), subobjects); builder.enabled = this.enabled; builder.dynamic = this.dynamic; return builder; @@ -429,7 +429,7 @@ public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { * This is typically used in the context of a mapper merge when there's not enough budget to add the entire object. */ ObjectMapper withoutMappers() { - return new ObjectMapper(simpleName(), fullPath, enabled, subobjects, storeArraySource, dynamic, Map.of()); + return new ObjectMapper(leafName(), fullPath, enabled, subobjects, storeArraySource, dynamic, Map.of()); } @Override @@ -500,7 +500,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex } var mergeResult = MergeResult.build(this, (ObjectMapper) mergeWith, parentMergeContext); return new ObjectMapper( - simpleName(), + leafName(), fullPath, mergeResult.enabled, mergeResult.subObjects, @@ -559,7 +559,7 @@ static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, Ma } else { trackArraySource = existing.storeArraySource; } - MapperMergeContext objectMergeContext = existing.createChildContext(parentMergeContext, existing.simpleName()); + MapperMergeContext objectMergeContext = existing.createChildContext(parentMergeContext, existing.leafName()); Map mergedMappers = buildMergedMappers(existing, mergeWithObject, objectMergeContext, subObjects.value()); return new MergeResult( enabled, @@ -581,13 +581,13 @@ private static Map buildMergedMappers( if (subobjects == false && childOfExistingMapper instanceof ObjectMapper objectMapper) { // An existing mapping with sub-objects is merged with a mapping that has set `subobjects: false` objectMapper.asFlattenedFieldMappers(objectMergeContext.getMapperBuilderContext()) - .forEach(m -> mergedMappers.put(m.simpleName(), m)); + .forEach(m -> mergedMappers.put(m.leafName(), m)); } else { putMergedMapper(mergedMappers, childOfExistingMapper); } } for (Mapper mergeWithMapper : mergeWithObject) { - Mapper mergeIntoMapper = mergedMappers.get(mergeWithMapper.simpleName()); + Mapper mergeIntoMapper = mergedMappers.get(mergeWithMapper.leafName()); if (mergeIntoMapper == null) { if (subobjects == false && mergeWithMapper instanceof ObjectMapper objectMapper) { // An existing mapping that has set `subobjects: false` is merged with a mapping with sub-objects @@ -625,7 +625,7 @@ private static Map buildMergedMappers( private static void putMergedMapper(Map mergedMappers, @Nullable Mapper merged) { if (merged != null) { - mergedMappers.put(merged.simpleName(), merged); + mergedMappers.put(merged.leafName(), merged); } } @@ -656,11 +656,11 @@ List asFlattenedFieldMappers(MapperBuilderContext context) { private void asFlattenedFieldMappers(MapperBuilderContext context, List flattenedMappers, ContentPath path) { ensureFlattenable(context, path); - path.add(simpleName()); + path.add(leafName()); for (Mapper mapper : mappers.values()) { if (mapper instanceof FieldMapper fieldMapper) { FieldMapper.Builder fieldBuilder = fieldMapper.getMergeBuilder(); - fieldBuilder.setName(path.pathAsText(mapper.simpleName())); + fieldBuilder.setLeafName(path.pathAsText(mapper.leafName())); flattenedMappers.add(fieldBuilder.build(context)); } else if (mapper instanceof ObjectMapper objectMapper) { objectMapper.asFlattenedFieldMappers(context, flattenedMappers, path); @@ -691,10 +691,10 @@ private void ensureFlattenable(MapperBuilderContext context, ContentPath path) { private void throwAutoFlatteningException(ContentPath path, String reason) { throw new IllegalArgumentException( "Object mapper [" - + path.pathAsText(simpleName()) + + path.pathAsText(leafName()) + "] was found in a context where subobjects is set to false. " + "Auto-flattening [" - + path.pathAsText(simpleName()) + + path.pathAsText(leafName()) + "] failed because " + reason ); @@ -707,7 +707,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } void toXContent(XContentBuilder builder, Params params, ToXContent custom) throws IOException { - builder.startObject(simpleName()); + builder.startObject(leafName()); if (mappers.isEmpty() && custom == null) { // only write the object content type if there are no properties, otherwise, it is automatically detected builder.field("type", CONTENT_TYPE); @@ -849,7 +849,7 @@ public void write(XContentBuilder b) throws IOException { if (isRoot()) { b.startObject(); } else { - b.startObject(simpleName()); + b.startObject(leafName()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index df3c6c54b27fd..413a645cebce6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -77,11 +77,11 @@ public PassThroughObjectMapper.Builder setPriority(int priority) { @Override public PassThroughObjectMapper build(MapperBuilderContext context) { return new PassThroughObjectMapper( - name(), - context.buildFullName(name()), + leafName(), + context.buildFullName(leafName()), enabled, dynamic, - buildMappers(context.createChildContext(name(), timeSeriesDimensionSubFields.value(), dynamic)), + buildMappers(context.createChildContext(leafName(), timeSeriesDimensionSubFields.value(), dynamic)), timeSeriesDimensionSubFields, priority ); @@ -113,7 +113,7 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { @Override PassThroughObjectMapper withoutMappers() { - return new PassThroughObjectMapper(simpleName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields, priority); + return new PassThroughObjectMapper(leafName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields, priority); } @Override @@ -131,7 +131,7 @@ public int priority() { @Override public PassThroughObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(simpleName()); + PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(leafName()); builder.enabled = enabled; builder.dynamic = dynamic; builder.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; @@ -153,7 +153,7 @@ public PassThroughObjectMapper merge(Mapper mergeWith, MapperMergeContext parent : this.timeSeriesDimensionSubFields; return new PassThroughObjectMapper( - simpleName(), + leafName(), fullPath(), mergeResult.enabled(), mergeResult.dynamic(), @@ -165,7 +165,7 @@ public PassThroughObjectMapper merge(Mapper mergeWith, MapperMergeContext parent @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(simpleName()); + builder.startObject(leafName()); builder.field("type", CONTENT_TYPE); if (timeSeriesDimensionSubFields.explicit()) { builder.field(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM, timeSeriesDimensionSubFields.value()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java index 67260273bc5a5..4a12ed77b4f26 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java @@ -90,8 +90,8 @@ protected Parameter[] getParameters() { @Override public PlaceHolderFieldMapper build(MapperBuilderContext context) { - PlaceHolderFieldType mappedFieldType = new PlaceHolderFieldType(context.buildFullName(name()), type, Map.of()); - return new PlaceHolderFieldMapper(name(), mappedFieldType, multiFieldsBuilder.build(this, context), copyTo, unknownParams); + PlaceHolderFieldType mappedFieldType = new PlaceHolderFieldType(context.buildFullName(leafName()), type, Map.of()); + return new PlaceHolderFieldMapper(leafName(), mappedFieldType, multiFieldsBuilder.build(this, context), copyTo, unknownParams); } } @@ -277,7 +277,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new PlaceHolderFieldMapper.Builder(simpleName(), typeName()).init(this); + return new PlaceHolderFieldMapper.Builder(leafName(), typeName()).init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 2e826c1294d60..e88131bc18e3a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -120,12 +120,12 @@ protected Parameter[] getParameters() { } protected RangeFieldType setupFieldType(MapperBuilderContext context) { - String fullName = context.buildFullName(name()); + String fullName = context.buildFullName(leafName()); if (format.isConfigured()) { if (type != RangeType.DATE) { throw new IllegalArgumentException( "field [" - + name() + + leafName() + "] of type [range]" + " should not define a dateTimeFormatter unless it is a " + RangeType.DATE @@ -167,7 +167,7 @@ protected RangeFieldType setupFieldType(MapperBuilderContext context) { @Override public RangeFieldMapper build(MapperBuilderContext context) { RangeFieldType ft = setupFieldType(context); - return new RangeFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, type, this); + return new RangeFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, type, this); } } @@ -364,7 +364,7 @@ boolean coerce() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), type, coerceByDefault).init(this); + return new Builder(leafName(), type, coerceByDefault).init(this); } @Override @@ -488,11 +488,11 @@ protected void writeValue(XContentBuilder b, BytesRef value) throws IOException case 0: return; case 1: - b.field(simpleName()); + b.field(leafName()); ranges.get(0).toXContent(b, fieldType().dateTimeFormatter); break; default: - b.startArray(simpleName()); + b.startArray(leafName()); for (var range : ranges) { range.toXContent(b, fieldType().dateTimeFormatter); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index ea00901bf681f..4bfce65a8bc92 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -108,7 +108,7 @@ public RootObjectMapper.Builder addRuntimeFields(Map runti @Override public RootObjectMapper build(MapperBuilderContext context) { return new RootObjectMapper( - name(), + leafName(), enabled, subobjects, storeArraySource, @@ -161,7 +161,7 @@ public RootObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { @Override RootObjectMapper withoutMappers() { return new RootObjectMapper( - simpleName(), + leafName(), enabled, subobjects, storeArraySource, @@ -277,7 +277,7 @@ public RootObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeCo } return new RootObjectMapper( - simpleName(), + leafName(), mergeResult.enabled(), mergeResult.subObjects(), mergeResult.trackArraySource(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 04d158c5ee99c..f8f4828c5d003 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -374,18 +374,18 @@ private TextFieldType buildFieldType( if (analyzers.positionIncrementGap.isConfigured()) { if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException( - "Cannot set position_increment_gap on field [" + name() + "] without positions enabled" + "Cannot set position_increment_gap on field [" + leafName() + "] without positions enabled" ); } } TextSearchInfo tsi = new TextSearchInfo(fieldType, similarity.getValue(), searchAnalyzer, searchQuoteAnalyzer); TextFieldType ft; if (indexCreatedVersion.isLegacyIndexVersion()) { - ft = new LegacyTextFieldType(context.buildFullName(name()), index.getValue(), store.getValue(), tsi, meta.getValue()); + ft = new LegacyTextFieldType(context.buildFullName(leafName()), index.getValue(), store.getValue(), tsi, meta.getValue()); // ignore fieldData and eagerGlobalOrdinals } else { ft = new TextFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), index.getValue(), store.getValue(), tsi, @@ -407,7 +407,7 @@ private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, FieldType fie return null; } if (index.getValue() == false) { - throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]"); + throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + leafName() + "]"); } /* * Mappings before v7.2.1 use {@link Builder#name} instead of {@link Builder#fullName} @@ -416,7 +416,7 @@ private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, FieldType fie * or a multi-field). This way search will continue to work on old indices and new indices * will use the expected full name. */ - String fullName = indexCreatedVersion.before(IndexVersions.V_7_2_1) ? name() : context.buildFullName(name()); + String fullName = indexCreatedVersion.before(IndexVersions.V_7_2_1) ? leafName() : context.buildFullName(leafName()); // Copy the index options of the main field to allow phrase queries on // the prefix field. FieldType pft = new FieldType(fieldType); @@ -448,10 +448,10 @@ private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent) return null; } if (index.get() == false) { - throw new IllegalArgumentException("Cannot set index_phrases on unindexed field [" + name() + "]"); + throw new IllegalArgumentException("Cannot set index_phrases on unindexed field [" + leafName() + "]"); } if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - throw new IllegalArgumentException("Cannot set index_phrases on field [" + name() + "] if positions are not enabled"); + throw new IllegalArgumentException("Cannot set index_phrases on field [" + leafName() + "] if positions are not enabled"); } FieldType phraseFieldType = new FieldType(fieldType); PhraseWrappedAnalyzer a = new PhraseWrappedAnalyzer( @@ -480,7 +480,7 @@ public TextFieldMapper build(MapperBuilderContext context) { throw new MapperParsingException("Cannot use reserved field name [" + mapper.name() + "]"); } } - return new TextFieldMapper(name(), fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo, this); + return new TextFieldMapper(leafName(), fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo, this); } } @@ -1265,7 +1265,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabledViaIndexMode).init(this); + return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabledViaIndexMode).init(this); } @Override @@ -1459,7 +1459,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { ); } if (store) { - return new StringStoredFieldFieldLoader(name(), simpleName(), null) { + return new StringStoredFieldFieldLoader(name(), leafName(), null) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); @@ -1469,7 +1469,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { var kwd = SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return kwd.syntheticFieldLoader(simpleName()); + return kwd.syntheticFieldLoader(leafName()); } throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 024cc5cb721e6..4666c6f7427c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -202,13 +202,13 @@ protected Parameter[] getParameters() { public FlattenedFieldMapper build(MapperBuilderContext context) { MultiFields multiFields = multiFieldsBuilder.build(this, context); if (multiFields.iterator().hasNext()) { - throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name() + "] does not support [fields]"); + throw new IllegalArgumentException(CONTENT_TYPE + " field [" + leafName() + "] does not support [fields]"); } if (copyTo.copyToFields().isEmpty() == false) { - throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name() + "] does not support [copy_to]"); + throw new IllegalArgumentException(CONTENT_TYPE + " field [" + leafName() + "] does not support [copy_to]"); } MappedFieldType ft = new RootFlattenedFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get(), hasDocValues.get(), meta.get(), @@ -216,7 +216,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { eagerGlobalOrdinals.get(), dimensions.get() ); - return new FlattenedFieldMapper(name(), ft, this); + return new FlattenedFieldMapper(leafName(), ft, this); } } @@ -801,7 +801,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override @@ -815,7 +815,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { return SourceLoader.SyntheticFieldLoader.NOTHING; } if (fieldType().hasDocValues()) { - return new FlattenedSortedSetDocValuesSyntheticFieldLoader(name() + "._keyed", simpleName()); + return new FlattenedSortedSetDocValuesSyntheticFieldLoader(name() + "._keyed", leafName()); } throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 7d385c189479b..beb64973ff0fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -245,9 +245,9 @@ public Builder dimensions(int dimensions) { @Override public DenseVectorFieldMapper build(MapperBuilderContext context) { return new DenseVectorFieldMapper( - name(), + leafName(), new DenseVectorFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexVersionCreated, elementType.getValue(), dims.getValue(), @@ -1666,7 +1666,7 @@ public void parse(DocumentParserContext context) throws IOException { fieldType().meta() ); Mapper update = new DenseVectorFieldMapper( - simpleName(), + leafName(), updatedDenseVectorFieldType, indexOptions, indexCreatedVersion, @@ -1757,7 +1757,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexCreatedVersion).init(this); + return new Builder(leafName(), indexCreatedVersion).init(this); } private static IndexOptions parseIndexOptions(String fieldName, Object propNode) { @@ -1883,7 +1883,7 @@ public void write(XContentBuilder b) throws IOException { if (hasMagnitude) { magnitude = Float.intBitsToFloat((int) magnitudeReader.longValue()); } - b.startArray(simpleName()); + b.startArray(leafName()); if (values != null) { for (float v : values.vectorValue()) { if (hasMagnitude) { @@ -1943,7 +1943,7 @@ public void write(XContentBuilder b) throws IOException { if (false == hasValue) { return; } - b.startArray(simpleName()); + b.startArray(leafName()); BytesRef ref = values.binaryValue(); ByteBuffer byteBuffer = ByteBuffer.wrap(ref.bytes, ref.offset, ref.length); if (indexCreatedVersion.onOrAfter(LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index e07c9247072b9..593a45c0c325a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -67,8 +67,8 @@ protected Parameter[] getParameters() { @Override public SparseVectorFieldMapper build(MapperBuilderContext context) { return new SparseVectorFieldMapper( - name(), - new SparseVectorFieldType(context.buildFullName(name()), meta.getValue()), + leafName(), + new SparseVectorFieldType(context.buildFullName(leafName()), meta.getValue()), multiFieldsBuilder.build(this, context), copyTo ); @@ -142,7 +142,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 453d0b3201560..37b42fd171a5a 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1739,6 +1739,7 @@ private static IndexMetadata.Builder restoreOverClosedIndex(IndexMetadata snapsh .state(IndexMetadata.State.OPEN) .version(Math.max(snapshotIndexMetadata.getVersion(), 1 + currentIndexMetadata.getVersion())) .mappingVersion(Math.max(snapshotIndexMetadata.getMappingVersion(), 1 + currentIndexMetadata.getMappingVersion())) + .mappingsUpdatedVersion(snapshotIndexMetadata.getMappingsUpdatedVersion()) .settingsVersion(Math.max(snapshotIndexMetadata.getSettingsVersion(), 1 + currentIndexMetadata.getSettingsVersion())) .aliasesVersion(Math.max(snapshotIndexMetadata.getAliasesVersion(), 1 + currentIndexMetadata.getAliasesVersion())) .timestampRange(IndexLongFieldRange.NO_SHARDS) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 70c33283c7475..440ac3f286878 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -186,6 +186,7 @@ public void testToXContentWithDeprecatedClusterState() { "0": [] }, "rollover_info": {}, + "mappings_updated_version" : %s, "system": false, "timestamp_range": { "shards": [] @@ -213,6 +214,7 @@ public void testToXContentWithDeprecatedClusterState() { Version.CURRENT, IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), + IndexVersion.current(), IndexVersion.current() ), """ @@ -225,7 +227,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { assertXContent( createClusterRerouteResponse(createClusterState()), new ToXContent.MapParams(Map.of("metric", "metadata", "settings_filter", "index.number*,index.version.created")), - """ + Strings.format(""" { "acknowledged" : true, "state" : { @@ -265,6 +267,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "0" : [ ] }, "rollover_info" : { }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -277,7 +280,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "reserved_state":{} } } - }""", + }""", IndexVersion.current()), """ The [state] field in the response to the reroute API is deprecated and will be removed in a future version. \ Specify ?metric=none to adopt the future behaviour.""" diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index ee98f40a6cb29..70bdfd6072a27 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -309,6 +309,7 @@ public void testToXContent() throws IOException { "time": 1 } }, + "mappings_updated_version" : %s, "system": false, "timestamp_range": { "shards": [] @@ -385,6 +386,7 @@ public void testToXContent() throws IOException { TransportVersion.current(), IndexVersion.current(), IndexVersion.current(), + IndexVersion.current(), allocationId, allocationId ) @@ -572,6 +574,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "time" : 1 } }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -652,6 +655,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti TransportVersion.current(), IndexVersion.current(), IndexVersion.current(), + IndexVersion.current(), allocationId, allocationId ), @@ -845,6 +849,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "time" : 1 } }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -925,6 +930,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti TransportVersion.current(), IndexVersion.current(), IndexVersion.current(), + IndexVersion.current(), allocationId, allocationId ), @@ -1014,6 +1020,7 @@ public void testToXContentSameTypeName() throws IOException { "0" : [ ] }, "rollover_info" : { }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -1032,7 +1039,7 @@ public void testToXContentSameTypeName() throws IOException { "unassigned" : [ ], "nodes" : { } } - }""", IndexVersion.current()), Strings.toString(builder)); + }""", IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); } public void testNodeFeaturesSorted() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index 116acf938fcbc..4a19284f3c4f9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -80,6 +81,7 @@ public void testIndexMetadataSerialization() throws IOException { Map customMap = new HashMap<>(); customMap.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); customMap.put(randomAlphaOfLength(10), randomAlphaOfLength(15)); + IndexVersion mappingsUpdatedVersion = IndexVersionUtils.randomVersion(); IndexMetadataStats indexStats = randomBoolean() ? randomIndexStats(numShard) : null; Double indexWriteLoadForecast = randomBoolean() ? randomDoubleBetween(0.0, 128, true) : null; Long shardSizeInBytesForecast = randomBoolean() ? randomLongBetween(1024, 10240) : null; @@ -106,6 +108,7 @@ public void testIndexMetadataSerialization() throws IOException { randomNonNegativeLong() ) ) + .mappingsUpdatedVersion(mappingsUpdatedVersion) .stats(indexStats) .indexWriteLoadForecast(indexWriteLoadForecast) .shardSizeInBytesForecast(shardSizeInBytesForecast) @@ -137,6 +140,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata.getCreationDate(), fromXContentMeta.getCreationDate()); assertEquals(metadata.getRoutingFactor(), fromXContentMeta.getRoutingFactor()); assertEquals(metadata.primaryTerm(0), fromXContentMeta.primaryTerm(0)); + assertEquals(metadata.getMappingsUpdatedVersion(), fromXContentMeta.getMappingsUpdatedVersion()); assertEquals(metadata.isSystem(), fromXContentMeta.isSystem()); Map expectedCustom = Map.of("my_custom", new DiffableStringMap(customMap)); assertEquals(metadata.getCustomData(), expectedCustom); @@ -164,6 +168,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata.getRolloverInfos(), deserialized.getRolloverInfos()); assertEquals(deserialized.getCustomData(), expectedCustom); assertEquals(metadata.getCustomData(), deserialized.getCustomData()); + assertEquals(metadata.getMappingsUpdatedVersion(), deserialized.getMappingsUpdatedVersion()); assertEquals(metadata.isSystem(), deserialized.isSystem()); assertEquals(metadata.getStats(), deserialized.getStats()); assertEquals(metadata.getForecastedWriteLoad(), deserialized.getForecastedWriteLoad()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java index 5fa5ebbaf5d06..1fc7807bd9735 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -94,6 +95,7 @@ public void testMappingVersion() throws Exception { singleTask(request) ); assertThat(resultingState.metadata().index("test").getMappingVersion(), equalTo(1 + previousVersion)); + assertThat(resultingState.metadata().index("test").getMappingsUpdatedVersion(), equalTo(IndexVersion.current())); } public void testMappingVersionUnchanged() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index 3a522f3f5c06c..cda9ae6c84a55 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -349,6 +349,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { "0" : [ ] }, "rollover_info" : { }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -360,7 +361,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { }, "reserved_state" : { } } - }""", IndexVersion.current()), Strings.toString(builder)); + }""", IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); } public void testToXContentGateway_FlatSettingFalse_ReduceMappingTrue() throws IOException { @@ -519,6 +520,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce "time" : 1 } }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -530,7 +532,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce }, "reserved_state" : { } } - }""", IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); + }""", IndexVersion.current(), IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); } public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOException { @@ -629,6 +631,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce "time" : 1 } }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -640,7 +643,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce }, "reserved_state" : { } } - }""", IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); + }""", IndexVersion.current(), IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); } public void testToXContentAPIReservedMetadata() throws IOException { @@ -765,6 +768,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { "time" : 1 } }, + "mappings_updated_version" : %s, "system" : false, "timestamp_range" : { "shards" : [ ] @@ -821,7 +825,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { } } } - }""", IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); + }""", IndexVersion.current(), IndexVersion.current(), IndexVersion.current()), Strings.toString(builder)); } private Metadata buildMetadata() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index d417d6c647d05..e6e06d331a0c7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2606,7 +2606,7 @@ same name need to be part of the same mappings (hence the same document). If th Mapper baz = ((ObjectMapper) barMapper).getMapper("baz"); assertNotNull(baz); assertEquals("foo.bar.baz", baz.name()); - assertEquals("baz", baz.simpleName()); + assertEquals("baz", baz.leafName()); List fields = doc.rootDoc().getFields("foo.bar.baz"); assertEquals(2, fields.size()); String[] fieldStrings = fields.stream().map(Object::toString).toArray(String[]::new); @@ -3245,7 +3245,7 @@ protected String contentType() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new StringStoredFieldFieldLoader(name(), simpleName(), null) { + return new StringStoredFieldFieldLoader(name(), leafName(), null) { @Override protected void write(XContentBuilder b, Object value) throws IOException { BytesRef ref = (BytesRef) value; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index bb5d50267642f..3b46f15b33cc7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -273,7 +273,7 @@ public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { DocumentMapper documentMapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_RECOVERY); - assertEquals(testString, documentMapper.mappers().getMapper(testString).simpleName()); + assertEquals(testString, documentMapper.mappers().getMapper(testString).leafName()); } public void testIsMetadataField() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java index 5b636c985f695..e3da8b39e1ec6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java @@ -114,7 +114,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public Builder getMergeBuilder() { - return new Builder(simpleName()); + return new Builder(leafName()); } @Override @@ -135,7 +135,7 @@ protected Parameter[] getParameters() { @Override public FieldMapper build(MapperBuilderContext context) { - return new TestInferenceFieldMapper(name()); + return new TestInferenceFieldMapper(leafName()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index aa22a345c5cec..ddc4b4b7dc674 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -206,19 +206,19 @@ to get the wrong path (missing the first portion). assertEquals(1, mapping.getRoot().mappers.size()); Mapper object = mapping.getRoot().getMapper("obj"); assertThat(object, CoreMatchers.instanceOf(ObjectMapper.class)); - assertEquals("obj", object.simpleName()); + assertEquals("obj", object.leafName()); assertEquals("obj", object.name()); ObjectMapper objectMapper = (ObjectMapper) object; assertEquals(1, objectMapper.mappers.size()); object = objectMapper.getMapper("source"); assertThat(object, CoreMatchers.instanceOf(ObjectMapper.class)); - assertEquals("source", object.simpleName()); + assertEquals("source", object.leafName()); assertEquals("obj.source", object.name()); objectMapper = (ObjectMapper) object; assertEquals(1, objectMapper.mappers.size()); object = objectMapper.getMapper("geo"); assertThat(object, CoreMatchers.instanceOf(ObjectMapper.class)); - assertEquals("geo", object.simpleName()); + assertEquals("geo", object.leafName()); assertEquals("obj.source.geo", object.name()); objectMapper = (ObjectMapper) object; assertEquals(1, objectMapper.mappers.size()); @@ -226,7 +226,7 @@ to get the wrong path (missing the first portion). assertThat(location, CoreMatchers.instanceOf(GeoPointFieldMapper.class)); GeoPointFieldMapper geoPointFieldMapper = (GeoPointFieldMapper) location; assertEquals("obj.source.geo.location", geoPointFieldMapper.name()); - assertEquals("location", geoPointFieldMapper.simpleName()); + assertEquals("location", geoPointFieldMapper.leafName()); assertEquals("obj.source.geo.location", geoPointFieldMapper.mappedFieldType.name()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTests.java index 7b8486e5050c2..8c64868045490 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTests.java @@ -64,7 +64,7 @@ protected Parameter[] getParameters() { @Override public NonDynamicFieldMapper build(MapperBuilderContext context) { - return new NonDynamicFieldMapper(name(), new TextFieldMapper.TextFieldType(name(), false, true, meta.getValue())); + return new NonDynamicFieldMapper(leafName(), new TextFieldMapper.TextFieldType(leafName(), false, true, meta.getValue())); } } @@ -82,7 +82,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 25ef3c8550ec0..b76949b8a8e4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -53,10 +53,10 @@ public void testMerge() { final ObjectMapper mergedFoo = (ObjectMapper) merged.getMapper("foo"); { Mapper bar = mergedFoo.getMapper("bar"); - assertEquals("bar", bar.simpleName()); + assertEquals("bar", bar.leafName()); assertEquals("foo.bar", bar.name()); Mapper baz = mergedFoo.getMapper("baz"); - assertEquals("baz", baz.simpleName()); + assertEquals("baz", baz.leafName()); assertEquals("foo.baz", baz.name()); } } @@ -140,7 +140,7 @@ public void testMergedFieldNamesFieldWithDotsSubobjectsFalseAtRoot() { final KeywordFieldMapper keywordFieldMapper = (KeywordFieldMapper) merged.getMapper("host.name"); assertEquals("host.name", keywordFieldMapper.name()); - assertEquals("host.name", keywordFieldMapper.simpleName()); + assertEquals("host.name", keywordFieldMapper.leafName()); } public void testMergedFieldNamesFieldWithDotsSubobjectsFalse() { @@ -157,7 +157,7 @@ public void testMergedFieldNamesFieldWithDotsSubobjectsFalse() { ObjectMapper metrics = (ObjectMapper) foo.getMapper("metrics"); final KeywordFieldMapper keywordFieldMapper = (KeywordFieldMapper) metrics.getMapper("host.name"); assertEquals("foo.metrics.host.name", keywordFieldMapper.name()); - assertEquals("host.name", keywordFieldMapper.simpleName()); + assertEquals("host.name", keywordFieldMapper.leafName()); } public void testMergedFieldNamesMultiFields() { @@ -170,10 +170,10 @@ public void testMergedFieldNamesMultiFields() { TextFieldMapper text = (TextFieldMapper) merged.getMapper("text"); assertEquals("text", text.name()); - assertEquals("text", text.simpleName()); + assertEquals("text", text.leafName()); KeywordFieldMapper keyword = (KeywordFieldMapper) text.multiFields().iterator().next(); assertEquals("text.keyword", keyword.name()); - assertEquals("keyword", keyword.simpleName()); + assertEquals("keyword", keyword.leafName()); } public void testMergedFieldNamesMultiFieldsWithinSubobjectsFalse() { @@ -190,10 +190,10 @@ public void testMergedFieldNamesMultiFieldsWithinSubobjectsFalse() { ObjectMapper metrics = (ObjectMapper) foo.getMapper("metrics"); final TextFieldMapper textFieldMapper = (TextFieldMapper) metrics.getMapper("host.name"); assertEquals("foo.metrics.host.name", textFieldMapper.name()); - assertEquals("host.name", textFieldMapper.simpleName()); + assertEquals("host.name", textFieldMapper.leafName()); FieldMapper fieldMapper = textFieldMapper.multiFields.iterator().next(); assertEquals("foo.metrics.host.name.keyword", fieldMapper.name()); - assertEquals("keyword", fieldMapper.simpleName()); + assertEquals("keyword", fieldMapper.leafName()); } public void testMergeWithLimit() { @@ -324,7 +324,7 @@ public void testMergeSubobjectsFalseWithObject() { private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { FieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); FieldMapper fieldMapper = fieldBuilder.build(MapperBuilderContext.root(false, false)); - assertEquals("host.name", fieldMapper.simpleName()); + assertEquals("host.name", fieldMapper.leafName()); assertEquals("host.name", fieldMapper.name()); return new RootObjectMapper.Builder("_doc", Explicit.EXPLICIT_FALSE).add(fieldBuilder) .build(MapperBuilderContext.root(false, false)); @@ -342,7 +342,7 @@ private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { MapperService.MergeReason.MAPPING_UPDATE ) ); - assertEquals("host.name", fieldMapper.simpleName()); + assertEquals("host.name", fieldMapper.leafName()); assertEquals("foo.metrics.host.name", fieldMapper.name()); return new ObjectMapper.Builder("foo", ObjectMapper.Defaults.SUBOBJECTS).add( new ObjectMapper.Builder("metrics", Explicit.EXPLICIT_FALSE).add(fieldBuilder) @@ -361,10 +361,10 @@ private ObjectMapper.Builder createObjectSubobjectsFalseLeafWithMultiField() { MapperService.MergeReason.MAPPING_UPDATE ) ); - assertEquals("host.name", textKeywordMultiField.simpleName()); + assertEquals("host.name", textKeywordMultiField.leafName()); assertEquals("foo.metrics.host.name", textKeywordMultiField.name()); FieldMapper fieldMapper = textKeywordMultiField.multiFields.iterator().next(); - assertEquals("keyword", fieldMapper.simpleName()); + assertEquals("keyword", fieldMapper.leafName()); assertEquals("foo.metrics.host.name.keyword", fieldMapper.name()); return new ObjectMapper.Builder("foo", ObjectMapper.Defaults.SUBOBJECTS).add( new ObjectMapper.Builder("metrics", Explicit.EXPLICIT_FALSE).add(fieldBuilder) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 308f775ec7b28..55761e5ec339d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -176,7 +176,7 @@ protected Parameter[] getParameters() { @Override public FieldMapper build(MapperBuilderContext context) { - return new TestMapper(name(), context.buildFullName(name()), multiFieldsBuilder.build(this, context), copyTo, this); + return new TestMapper(leafName(), context.buildFullName(leafName()), multiFieldsBuilder.build(this, context), copyTo, this); } } @@ -228,7 +228,7 @@ protected TestMapper( @Override public Builder getMergeBuilder() { - return new ParametrizedMapperTests.Builder(simpleName()).init(this); + return new ParametrizedMapperTests.Builder(leafName()).init(this); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index b95619602573c..f207f3c4cd314 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -34,7 +34,7 @@ public MockFieldMapper(String fullName, MappedFieldType fieldType, MultiFields m @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()); + return new Builder(leafName()); } static String findSimpleName(String fullName) { @@ -92,7 +92,7 @@ public Builder copyTo(String field) { @Override public MockFieldMapper build(MapperBuilderContext context) { MultiFields multiFields = multiFieldsBuilder.build(this, context); - return new MockFieldMapper(name(), fieldType, multiFields, copyTo); + return new MockFieldMapper(leafName(), fieldType, multiFields, copyTo); } } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 9fe7fc647455e..90b50d3df34bd 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -97,8 +97,8 @@ protected Parameter[] getParameters() { @Override public HistogramFieldMapper build(MapperBuilderContext context) { return new HistogramFieldMapper( - name(), - new HistogramFieldType(context.buildFullName(name()), meta.getValue()), + leafName(), + new HistogramFieldType(context.buildFullName(leafName()), meta.getValue()), multiFieldsBuilder.build(this, context), copyTo, this @@ -138,7 +138,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), ignoreMalformedByDefault).init(this); + return new Builder(leafName(), ignoreMalformedByDefault).init(this); } @Override @@ -289,7 +289,7 @@ protected boolean supportsParsingObject() { @Override public void parse(DocumentParserContext context) throws IOException { - context.path().add(simpleName()); + context.path().add(leafName()); boolean shouldStoreMalformedDataForSyntheticSource = context.mappingLookup().isSourceSynthetic() && ignoreMalformed(); XContentParser.Token token; @@ -521,7 +521,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { } return new CompositeSyntheticFieldLoader( - simpleName(), + leafName(), name(), new HistogramSyntheticFieldLoader(), new CompositeSyntheticFieldLoader.MalformedValuesLayer(name()) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json deleted file mode 100644 index b02866e867c4a..0000000000000 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "mode": "logs", - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "mapping": { - "ignore_malformed": true, - "total_fields": { - "ignore_dynamic_beyond_limit": true - } - }, - "default_pipeline": "logs@default-pipeline" - } - } - }, - "_meta": { - "description": "default settings for the logs index template installed by x-pack", - "managed": true - }, - "version": ${xpack.stack.template.version}, - "deprecated": ${xpack.stack.template.deprecated} -} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index ca2659b8d8dea..240abf9934db5 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -5,6 +5,7 @@ "lifecycle": { "name": "logs" }, + "mode": "${xpack.stack.template.logs.index.mode}", "codec": "best_compression", "mapping": { "ignore_malformed": true, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 0fb35b4253d6d..5cae87850ae99 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -73,6 +73,7 @@ double pi() "geo_point|cartesian_point st_centroid_agg(field:geo_point|cartesian_point)" "boolean st_contains(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" "boolean st_disjoint(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" +"double st_distance(geomA:geo_point|cartesian_point, geomB:geo_point|cartesian_point)" "boolean st_intersects(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" "boolean st_within(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" "double st_x(point:geo_point|cartesian_point)" @@ -191,6 +192,7 @@ sqrt |number |"double|integer|long|unsigne st_centroid_ag|field |"geo_point|cartesian_point" |[""] st_contains |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] st_disjoint |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] +st_distance |[geomA, geomB] |["geo_point|cartesian_point", "geo_point|cartesian_point"] |[Expression of type `geo_point` or `cartesian_point`. If `null`\, the function returns `null`., Expression of type `geo_point` or `cartesian_point`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_point` and `cartesian_point` parameters.] st_intersects |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] st_within |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] st_x |point |"geo_point|cartesian_point" |Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. @@ -310,6 +312,7 @@ sqrt |Returns the square root of a number. The input can be any numeric st_centroid_ag|The centroid of a spatial field. st_contains |Returns whether the first geometry contains the second geometry. This is the inverse of the <> function. st_disjoint |Returns whether the two geometries or geometry columns are disjoint. This is the inverse of the <> function. In mathematical terms: ST_Disjoint(A, B) ⇔ A ⋂ B = ∅ +st_distance |Computes the distance between two points. For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. For geographic geometries, this is the circular distance along the great circle in meters. st_intersects |Returns true if two geometries intersect. They intersect if they have any point in common, including their interior points (points along lines or within polygons). This is the inverse of the <> function. In mathematical terms: ST_Intersects(A, B) ⇔ A ⋂ B ≠ ∅ st_within |Returns whether the first geometry is within the second geometry. This is the inverse of the <> function. st_x |Extracts the `x` coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the `longitude` value. @@ -430,6 +433,7 @@ sqrt |double st_centroid_ag|"geo_point|cartesian_point" |false |false |true st_contains |boolean |[false, false] |false |false st_disjoint |boolean |[false, false] |false |false +st_distance |double |[false, false] |false |false st_intersects |boolean |[false, false] |false |false st_within |boolean |[false, false] |false |false st_x |double |false |false |false @@ -487,5 +491,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added] meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -110 | 110 | 110 +111 | 111 | 111 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index bc24c2d23adc4..f3604a90a35c7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -146,7 +146,7 @@ c:long | x:double | y:double ; ############################################### -# Tests for ST_CENTROID on GEO_POINT type +# Tests for ST_CENTROID_AGG on GEO_POINT type centroidFromAirports required_capability: st_centroid_agg @@ -841,10 +841,10 @@ FROM airports // end::st_within-airports[] ; -// tag::st_within-airports-results[] +// tag::st_within-airports-result[] abbrev:keyword | city:keyword | city_location:geo_point | country:keyword | location:geo_point | name:text | scalerank:i | type:k HOD | Al Ḩudaydah | POINT(42.9511 14.8022) | Yemen | POINT(42.97109630194 14.7552534413725) | Hodeidah Int'l | 9 | mid -// end::st_within-airports-results[] +// end::st_within-airports-result[] ; airportCityLocationPointWithin @@ -870,6 +870,156 @@ location:geo_point | city_location:geo_point | count:long POINT (0 0) | POINT (0 0) | 1 ; +############################################### +# Tests for ST_DISTANCE with GEO_POINT + +literalGeoPointDistanceLiteralPoint +required_capability: st_distance + +ROW wkt = ["POINT(1 1)", "POINT(-1 -1)", "POINT(-1 1)", "POINT(1 -1)"] +| MV_EXPAND wkt +| EVAL pt = TO_GEOPOINT(wkt) +| EVAL distance = ST_DISTANCE(pt, TO_GEOPOINT("POINT(0 0)")) +; + +wkt:keyword | pt:geo_point | distance:double +"POINT(1 1)" | POINT(1 1) | 157249.59776811762 +"POINT(-1 -1)" | POINT(-1 -1) | 157249.59776811762 +"POINT(-1 1)" | POINT(-1 1) | 157249.59776811762 +"POINT(1 -1)" | POINT(1 -1) | 157249.59776811762 +; + +airportCityLocationPointDistance +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, city_location) +| STATS distance=AVG(distance), count=COUNT() +; + +distance:double | count:long +15869.987675939537 | 891 +; + +airportDistanceToCityCopenhagen +required_capability: st_distance + +// tag::st_distance-airports[] +FROM airports +| WHERE abbrev == "CPH" +| EVAL distance = ST_DISTANCE(location, city_location) +| KEEP abbrev, name, location, city_location, distance +// end::st_distance-airports[] +; + +// tag::st_distance-airports-result[] +abbrev:k | name:text | location:geo_point | city_location:geo_point | distance:d +CPH | Copenhagen | POINT(12.6493508684508 55.6285017221528) | POINT(12.5683 55.6761) | 7339.57266575626 +// end::st_distance-airports-result[] +; + +airportsWithinPolygonDistanceFromCopenhagenTrainStation +required_capability: st_distance + +FROM airports +| WHERE ST_WITHIN(location, TO_GEOSHAPE("POLYGON((12 40, 14 40, 14 60, 12 60, 12 40))")) +| EVAL distance = ROUND(ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)"))/1000,2) +| EVAL city_distance = ROUND(ST_DISTANCE(city_location, TO_GEOPOINT("POINT(12.565 55.673)"))/1000,2) +| KEEP abbrev, name, location, country, city, city_location, distance, city_distance +| SORT distance ASC +; + +abbrev:k | name:text | location:geo_point | country:k | city:k | city_location:geo_point | distance:d | city_distance:d +CPH | Copenhagen | POINT(12.6493508684508 55.6285017221528) | Denmark | Copenhagen | POINT(12.5683 55.6761) | 7.24 | 0.4 +GOT | Gothenburg | POINT(12.2938269092573 57.6857493534879) | Sweden | Gothenburg | POINT(11.9675 57.7075) | 224.42 | 229.15 +TXL | Berlin-Tegel Int'l | POINT(13.2903090925074 52.5544287044101) | Germany | Hohen Neuendorf | POINT(13.2833 52.6667) | 349.97 | 337.53 +DRS | Dresden | POINT(13.7649671440047 51.1250912428871) | Germany | Dresden | POINT(13.74 51.05) | 511.9 | 519.91 +VCE | Venice Marco Polo | POINT(12.3410673004369 45.5048477588455) | Italy | Mestre | POINT(12.2381 45.4906) | 1130.76 | 1132.46 +FCO | Leonardo da Vinci Int'l | POINT(12.2501008973638 41.7950786307394) | Italy | Fiumicino | POINT(12.2333 41.7667) | 1543.33 | 1546.5 +; + +airportsWithinDistanceFromCopenhagenTrainStation +required_capability: st_distance + +FROM airports +| WHERE ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) < 600000 +| EVAL distance = ROUND(ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)"))/1000,2) +| EVAL city_distance = ROUND(ST_DISTANCE(city_location, TO_GEOPOINT("POINT(12.565 55.673)"))/1000,2) +| KEEP abbrev, name, location, country, city, city_location, distance, city_distance +| SORT distance ASC +; + +abbrev:k | name:text | location:geo_point | country:k | city:k | city_location:geo_point | distance:d | city_distance:d +CPH | Copenhagen | POINT(12.6493508684508 55.6285017221528) | Denmark | Copenhagen | POINT(12.5683 55.6761) | 7.24 | 0.4 +GOT | Gothenburg | POINT(12.2938269092573 57.6857493534879) | Sweden | Gothenburg | POINT(11.9675 57.7075) | 224.42 | 229.15 +HAM | Hamburg | POINT(10.005647830925 53.6320011640866) | Germany | Norderstedt | POINT(10.0103 53.7064) | 280.34 | 273.42 +TXL | Berlin-Tegel Int'l | POINT(13.2903090925074 52.5544287044101) | Germany | Hohen Neuendorf | POINT(13.2833 52.6667) | 349.97 | 337.53 +BRE | Bremen | POINT(8.7858617703132 53.052287104156) | Germany | Bremen | POINT(8.8 53.0833) | 380.5 | 377.22 +NRK | Norrköping Airport | POINT(16.2339407695814 58.5833805017541) | Sweden | Norrköping | POINT(16.2 58.6) | 392.0 | 392.35 +GDN | Gdansk Lech Walesa | POINT(18.4684422165911 54.3807025352925) | Poland | Gdańsk | POINT(18.6453 54.3475) | 402.61 | 414.59 +NYO | Stockholm-Skavsta | POINT(16.9216055584254 58.7851041303448) | Sweden | Nyköping | POINT(17.0086 58.7531) | 433.99 | 434.43 +OSL | Oslo Gardermoen | POINT(11.0991032762581 60.1935783171386) | Norway | Oslo | POINT(10.7389 59.9133) | 510.03 | 483.71 +DRS | Dresden | POINT(13.7649671440047 51.1250912428871) | Germany | Dresden | POINT(13.74 51.05) | 511.9 | 519.91 +BMA | Bromma | POINT(17.9456175406145 59.3555902065112) | Sweden | Stockholm | POINT(18.0686 59.3294) | 520.18 | 522.54 +PLQ | Palanga Int'l | POINT(21.0974463986251 55.9713426235358) | Lithuania | Klaipėda | POINT(21.1667 55.75) | 533.67 | 538.56 +ARN | Arlanda | POINT(17.9307299016916 59.6511203397372) | Sweden | Stockholm | POINT(18.0686 59.3294) | 545.09 | 522.54 +SVG | Stavanger Sola | POINT (5.6298103297218 58.8821564842185) | Norway | Sandnes | POINT (5.7361 58.8517) | 548.26 | 541.35 +; + +airportsWithinDistanceCopenhagenTrainStationCount +required_capability: st_distance + +FROM airports +| WHERE ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) < 500000 +| STATS count=COUNT() BY country +| SORT count DESC, country ASC +; + +count:long | country:k +3 | Germany +3 | Sweden +1 | Denmark +1 | Poland +; + +airportsSortDistanceFromCopenhagenTrainStation +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) +| SORT distance ASC +| LIMIT 5 +| EVAL distance = ROUND(distance/1000,2) +| EVAL city_distance = ROUND(ST_DISTANCE(city_location, TO_GEOPOINT("POINT(12.565 55.673)"))/1000,2) +| KEEP abbrev, name, location, country, city, city_location, distance, city_distance +; + +abbrev:k | name:text | location:geo_point | country:k | city:k | city_location:geo_point | distance:d | city_distance:d +CPH | Copenhagen | POINT(12.6493508684508 55.6285017221528) | Denmark | Copenhagen | POINT(12.5683 55.6761) | 7.24 | 0.4 +GOT | Gothenburg | POINT(12.2938269092573 57.6857493534879) | Sweden | Gothenburg | POINT(11.9675 57.7075) | 224.42 | 229.15 +HAM | Hamburg | POINT(10.005647830925 53.6320011640866) | Germany | Norderstedt | POINT(10.0103 53.7064) | 280.34 | 273.42 +TXL | Berlin-Tegel Int'l | POINT(13.2903090925074 52.5544287044101) | Germany | Hohen Neuendorf | POINT(13.2833 52.6667) | 349.97 | 337.53 +BRE | Bremen | POINT(8.7858617703132 53.052287104156) | Germany | Bremen | POINT(8.8 53.0833) | 380.5 | 377.22 +; + +airportsSortDistanceFromAirportToCity +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, city_location) +| SORT distance ASC +| LIMIT 5 +| KEEP abbrev, name, location, country, city, city_location, distance +; + +abbrev:k | name:text | location:geo_point | country:k | city:k | city_location:geo_point | distance:d +TRD | Trondheim Vaernes | POINT(10.9168095241445 63.472029381717) | Norway | Stjørdalshalsen | POINT(10.9189 63.4712) | 138.86728011324072 +DHA | King Abdulaziz AB | POINT(50.1477245727844 26.2703680854768) | Saudi Arabia | Dhahran | POINT(50.15 26.2667) | 466.7314410344158 +NDB | Nouadhibou Int'l | POINT(-17.0334398691538 20.9290523064387) | Mauritania | Nouadhibou | POINT(-17.0333 20.9333) | 472.545954400989 +ESE | Ensenada | POINT(-116.595724400418 31.7977139760569) | Mexico | Rodolfo Sánchez Taboada | POINT(-116.5911 31.7958) | 486.1044856437723 +INU | Nauru Int'l | POINT(166.91613965882 -0.545037226856384) | Nauru | Yaren | POINT(166.9209 -0.5477) | 606.4888777331985 +; + ############################################### # Tests for Equality and casting with GEO_POINT @@ -1036,7 +1186,7 @@ ZAH | POINT (6779435.866395892 3436280.545331025) | Zahedan Int'l ; ############################################### -# Tests for ST_CENTROID on CARTESIAN_POINT type +# Tests for ST_CENTROID_AGG on CARTESIAN_POINT type cartesianCentroidFromAirports required_capability: st_centroid_agg @@ -1675,7 +1825,134 @@ POINT (4783520.5 1661010.0) | 1 ; ############################################### -# Tests for Equality and casting with GEO_POINT +# Tests for ST_DISTANCE with CARTESIAN_POINT + +literalCartesianPointDistanceLiteralPoint +required_capability: st_distance + +ROW wkt = ["POINT(1 1)", "POINT(-1 -1)", "POINT(-1 1)", "POINT(1 -1)"] +| MV_EXPAND wkt +| EVAL pt = TO_CARTESIANPOINT(wkt) +| EVAL distance = ST_DISTANCE(pt, TO_CARTESIANPOINT("POINT(0 0)")) +; + +wkt:keyword | pt:cartesian_point | distance:double +"POINT(1 1)" | POINT(1 1) | 1.4142135623730951 +"POINT(-1 -1)" | POINT(-1 -1) | 1.4142135623730951 +"POINT(-1 1)" | POINT(-1 1) | 1.4142135623730951 +"POINT(1 -1)" | POINT(1 -1) | 1.4142135623730951 +; + +airportCartesianCityLocationPointDistance +required_capability: st_distance + +FROM airports_web +| EVAL distance = ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)")) +| WHERE distance < 1000000 +| STATS distance=AVG(distance), min=min(distance), max=max(distance), count=COUNT() +; + +distance:double | min:double | max:double | count:long +676858.3463435044 | 7358.02077507206 | 971112.9731194031 | 12 +; + +airportCartesianDistanceToCityCopenhagen +required_capability: st_distance + +// tag::st_distance-airports_web[] +FROM airports_web +| WHERE abbrev == "CPH" +| EVAL distance = ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)")) +| KEEP abbrev, name, location, distance +// end::st_distance-airports_web[] +; + +// tag::st_distance-airports_web-result[] +abbrev:k | name:text | location:cartesian_point | distance:d +CPH | Copenhagen | POINT(1408119.2975413958 7484813.53657096) | 7358.02077507206 +// end::st_distance-airports_web-result[] +; + +airportsWithinPolygonCartesianDistanceFromCopenhagenTrainStation +required_capability: st_distance + +FROM airports_web +| WHERE ST_WITHIN(location, TO_CARTESIANPOINT("POLYGON((1300000 5000000, 1550000 5000000, 1550000 8000000, 1300000 8000000, 1300000 5000000))")) +| EVAL distance = ROUND(ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)"))/1000,2) +| KEEP abbrev, name, location, distance +| SORT distance ASC +; + +abbrev:k | name:text | location:cartesian_point | distance:d +CPH | Copenhagen | POINT(1408119.2975413958 7484813.53657096) | 7.36 +GOT | Gothenburg | POINT(1368542.5514391668 7901590.990691348) | 413.02 +TXL | Berlin-Tegel Int'l | POINT(1479470.4406631375 6901000.847325224) | 593.96 +DRS | Dresden | POINT(1532309.1332567444 6643450.815136505) | 856.38 +MUC | Franz-Josef-Strauss | POINT(1312241.1393453805 6165923.947863139) | 1327.18 +MUCf | Munich Freight Terminal | POINT(1310172.8388047176 6165247.042460089) | 1327.99 +VCE | Venice Marco Polo | POINT(1373801.3277301549 5701352.694091503) | 1788.88 +FCO | Leonardo da Vinci Int'l | POINT(1363674.994060762 5130332.471894705) | 2359.99 +; + +airportsWithinDistanceFromCopenhagenTrainStationCartesian +required_capability: st_distance + +FROM airports_web +| WHERE ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)")) < 1000000 +| EVAL distance = ROUND(ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)"))/1000,2) +| KEEP abbrev, name, location, distance +| SORT distance ASC +; + +abbrev:k | name:text | location:cartesian_point | distance:d +CPH | Copenhagen | POINT(1408119.2975413958 7484813.53657096) | 7.36 +GOT | Gothenburg | POINT(1368542.5514391668 7901590.990691348) | 413.02 +HAM | Hamburg | POINT(1113823.6215953932 7100767.499507231) | 484.84 +TXL | Berlin-Tegel Int'l | POINT(1479470.4406631375 6901000.847325224) | 593.96 +BRE | Bremen | POINT(978037.6584513546 6992675.481947904) | 654.09 +GDN | Gdansk Lech Walesa | POINT(2055897.583295918 7242589.051225524) | 698.3 +NRK | Norrköping Airport | POINT(1807154.0200379654 8090879.147781534) | 724.21 +NYO | Stockholm-Skavsta | POINT(1883704.5141685435 8134083.898609498) | 803.75 +DRS | Dresden | POINT(1532309.1332567444 6643450.815136505) | 856.38 +PLQ | Palanga Int'l | POINT(2348556.9901333293 7552712.896849933) | 947.73 +OSL | Oslo Gardermoen | POINT(1235546.524975006 8442962.64811417) | 967.55 +BMA | Bromma | POINT(1997697.0065920444 8257643.750388128) | 971.11 +; + +airportsWithinDistanceCopenhagenTrainStationCountCartesian +required_capability: st_distance + +FROM airports_web +| WHERE ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)")) < 1000000 +| STATS count=COUNT() +| SORT count DESC +; + +count:long +12 +; + +airportsSortDistanceFromCopenhagenTrainStationCartesian +required_capability: st_distance + +FROM airports_web +| EVAL distance = ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)")) +| SORT distance ASC +| LIMIT 5 +| EVAL distance = ROUND(distance/1000,2) +| KEEP abbrev, name, location, distance +; + +abbrev:k | name:text | location:cartesian_point | distance:d +CPH | Copenhagen | POINT(1408119.2975413958 7484813.53657096) | 7.36 +GOT | Gothenburg | POINT(1368542.5514391668 7901590.990691348) | 413.02 +HAM | Hamburg | POINT(1113823.6215953932 7100767.499507231) | 484.84 +TXL | Berlin-Tegel Int'l | POINT(1479470.4406631375 6901000.847325224) | 593.96 +BRE | Bremen | POINT(978037.6584513546 6992675.481947904) | 654.09 +; + +############################################### +# Tests for Equality and casting with CARTESIAN_POINT cartesianPointEquals required_capability: spatial_points_from_source diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java new file mode 100644 index 0000000000000..21b987f830a2c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceCartesianPointDocValuesAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final Point rightValue; + + private final DriverContext driverContext; + + public StDistanceCartesianPointDocValuesAndConstantEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { + LongVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock); + } + return eval(page.getPositionCount(), leftValueVector); + } + } + + public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue)); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, LongVector leftValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue)); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final Point rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + Point rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceCartesianPointDocValuesAndConstantEvaluator get(DriverContext context) { + return new StDistanceCartesianPointDocValuesAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + } + + @Override + public String toString() { + return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java new file mode 100644 index 0000000000000..13523eebdb2aa --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java @@ -0,0 +1,142 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceCartesianPointDocValuesAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final EvalOperator.ExpressionEvaluator rightValue; + + private final DriverContext driverContext; + + public StDistanceCartesianPointDocValuesAndSourceEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { + try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { + LongVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + BytesRefVector rightValueVector = rightValueBlock.asVector(); + if (rightValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + return eval(page.getPositionCount(), leftValueVector, rightValueVector).asBlock(); + } + } + } + + public DoubleBlock eval(int positionCount, LongBlock leftValueBlock, + BytesRefBlock rightValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rightValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rightValueBlock.getValueCount(p) != 1) { + if (rightValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendDouble(StDistance.processCartesianPointDocValuesAndSource(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); + } + return result.build(); + } + } + + public DoubleVector eval(int positionCount, LongVector leftValueVector, + BytesRefVector rightValueVector) { + try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendDouble(p, StDistance.processCartesianPointDocValuesAndSource(leftValueVector.getLong(p), rightValueVector.getBytesRef(p, rightValueScratch))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue, rightValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final EvalOperator.ExpressionEvaluator.Factory rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + EvalOperator.ExpressionEvaluator.Factory rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceCartesianPointDocValuesAndSourceEvaluator get(DriverContext context) { + return new StDistanceCartesianPointDocValuesAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + } + + @Override + public String toString() { + return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java new file mode 100644 index 0000000000000..23416e56788b6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java @@ -0,0 +1,131 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.io.IOException; +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceCartesianSourceAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final Point rightValue; + + private final DriverContext driverContext; + + public StDistanceCartesianSourceAndConstantEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { + BytesRefVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock); + } + return eval(page.getPositionCount(), leftValueVector); + } + } + + public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue)); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue)); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceCartesianSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final Point rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + Point rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceCartesianSourceAndConstantEvaluator get(DriverContext context) { + return new StDistanceCartesianSourceAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + } + + @Override + public String toString() { + return "StDistanceCartesianSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java new file mode 100644 index 0000000000000..537275d14d3a1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java @@ -0,0 +1,152 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.io.IOException; +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceCartesianSourceAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final EvalOperator.ExpressionEvaluator rightValue; + + private final DriverContext driverContext; + + public StDistanceCartesianSourceAndSourceEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { + try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { + BytesRefVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + BytesRefVector rightValueVector = rightValueBlock.asVector(); + if (rightValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + return eval(page.getPositionCount(), leftValueVector, rightValueVector); + } + } + } + + public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock, + BytesRefBlock rightValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rightValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rightValueBlock.getValueCount(p) != 1) { + if (rightValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processCartesianSourceAndSource(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector, + BytesRefVector rightValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processCartesianSourceAndSource(leftValueVector.getBytesRef(p, leftValueScratch), rightValueVector.getBytesRef(p, rightValueScratch))); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceCartesianSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue, rightValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final EvalOperator.ExpressionEvaluator.Factory rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + EvalOperator.ExpressionEvaluator.Factory rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceCartesianSourceAndSourceEvaluator get(DriverContext context) { + return new StDistanceCartesianSourceAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + } + + @Override + public String toString() { + return "StDistanceCartesianSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java new file mode 100644 index 0000000000000..3f96c8bf20ab7 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceGeoPointDocValuesAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final Point rightValue; + + private final DriverContext driverContext; + + public StDistanceGeoPointDocValuesAndConstantEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { + LongVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock); + } + return eval(page.getPositionCount(), leftValueVector); + } + } + + public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue)); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, LongVector leftValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue)); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final Point rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + Point rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceGeoPointDocValuesAndConstantEvaluator get(DriverContext context) { + return new StDistanceGeoPointDocValuesAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + } + + @Override + public String toString() { + return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java new file mode 100644 index 0000000000000..0bd1c4b6f5bd3 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java @@ -0,0 +1,151 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceGeoPointDocValuesAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final EvalOperator.ExpressionEvaluator rightValue; + + private final DriverContext driverContext; + + public StDistanceGeoPointDocValuesAndSourceEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { + try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { + LongVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + BytesRefVector rightValueVector = rightValueBlock.asVector(); + if (rightValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + return eval(page.getPositionCount(), leftValueVector, rightValueVector); + } + } + } + + public DoubleBlock eval(int positionCount, LongBlock leftValueBlock, + BytesRefBlock rightValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rightValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rightValueBlock.getValueCount(p) != 1) { + if (rightValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processGeoPointDocValuesAndSource(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, LongVector leftValueVector, + BytesRefVector rightValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processGeoPointDocValuesAndSource(leftValueVector.getLong(p), rightValueVector.getBytesRef(p, rightValueScratch))); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue, rightValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final EvalOperator.ExpressionEvaluator.Factory rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + EvalOperator.ExpressionEvaluator.Factory rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceGeoPointDocValuesAndSourceEvaluator get(DriverContext context) { + return new StDistanceGeoPointDocValuesAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + } + + @Override + public String toString() { + return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java new file mode 100644 index 0000000000000..556444ac8d740 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java @@ -0,0 +1,131 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.io.IOException; +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceGeoSourceAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final Point rightValue; + + private final DriverContext driverContext; + + public StDistanceGeoSourceAndConstantEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { + BytesRefVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock); + } + return eval(page.getPositionCount(), leftValueVector); + } + } + + public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue)); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue)); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceGeoSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final Point rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + Point rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceGeoSourceAndConstantEvaluator get(DriverContext context) { + return new StDistanceGeoSourceAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + } + + @Override + public String toString() { + return "StDistanceGeoSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java new file mode 100644 index 0000000000000..915c456a63f7c --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java @@ -0,0 +1,152 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import java.io.IOException; +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link StDistance}. + * This class is generated. Do not edit it. + */ +public final class StDistanceGeoSourceAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator leftValue; + + private final EvalOperator.ExpressionEvaluator rightValue; + + private final DriverContext driverContext; + + public StDistanceGeoSourceAndSourceEvaluator(Source source, + EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + DriverContext driverContext) { + this.leftValue = leftValue; + this.rightValue = rightValue; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { + try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { + BytesRefVector leftValueVector = leftValueBlock.asVector(); + if (leftValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + BytesRefVector rightValueVector = rightValueBlock.asVector(); + if (rightValueVector == null) { + return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); + } + return eval(page.getPositionCount(), leftValueVector, rightValueVector); + } + } + } + + public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock, + BytesRefBlock rightValueBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (leftValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (leftValueBlock.getValueCount(p) != 1) { + if (leftValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (rightValueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (rightValueBlock.getValueCount(p) != 1) { + if (rightValueBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendDouble(StDistance.processGeoSourceAndSource(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector, + BytesRefVector rightValueVector) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + BytesRef leftValueScratch = new BytesRef(); + BytesRef rightValueScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendDouble(StDistance.processGeoSourceAndSource(leftValueVector.getBytesRef(p, leftValueScratch), rightValueVector.getBytesRef(p, rightValueScratch))); + } catch (IllegalArgumentException | IOException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "StDistanceGeoSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(leftValue, rightValue); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory leftValue; + + private final EvalOperator.ExpressionEvaluator.Factory rightValue; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, + EvalOperator.ExpressionEvaluator.Factory rightValue) { + this.source = source; + this.leftValue = leftValue; + this.rightValue = rightValue; + } + + @Override + public StDistanceGeoSourceAndSourceEvaluator get(DriverContext context) { + return new StDistanceGeoSourceAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + } + + @Override + public String toString() { + return "StDistanceGeoSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 9b759a49eab4e..f30f7f84128c9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -82,7 +82,12 @@ public enum Cap { /** * Support multiple field mappings if appropriate conversion function is used (union types) */ - UNION_TYPES; + UNION_TYPES, + + /** + * Support for function {@code ST_DISTANCE}. Done in #108764. + */ + ST_DISTANCE; Cap() { snapshotOnly = false; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 7034f23be1662..9f5c156ef9554 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -95,6 +95,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialDisjoint; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StX; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StY; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; @@ -254,6 +255,7 @@ private FunctionDefinition[][] functions() { def(SpatialDisjoint.class, SpatialDisjoint::new, "st_disjoint"), def(SpatialIntersects.class, SpatialIntersects::new, "st_intersects"), def(SpatialWithin.class, SpatialWithin::new, "st_within"), + def(StDistance.class, StDistance::new, "st_distance"), def(StX.class, StX::new, "st_x"), def(StY.class, StY::new, "st_y") }, // conditional diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java new file mode 100644 index 0000000000000..0735c18e04e1e --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; + +import java.io.IOException; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; +import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; + +/** + * Spatial functions that take two arguments that must both be spatial types can inherit from this class. + * This provides common support for type resolution and validation. Ensuring that both arguments are spatial types + * and of compatible CRS. For example geo_point and geo_shape can be compared, but not geo_point and cartesian_point. + */ +public abstract class BinarySpatialFunction extends BinaryScalarFunction implements SpatialEvaluatorFactory.SpatialSourceResolution { + private final SpatialTypeResolver spatialTypeResolver; + protected SpatialCrsType crsType; + protected final boolean leftDocValues; + protected final boolean rightDocValues; + + protected BinarySpatialFunction( + Source source, + Expression left, + Expression right, + boolean leftDocValues, + boolean rightDocValues, + boolean pointsOnly + ) { + super(source, left, right); + this.leftDocValues = leftDocValues; + this.rightDocValues = rightDocValues; + this.spatialTypeResolver = new SpatialTypeResolver(this, pointsOnly); + } + + @Override + protected TypeResolution resolveType() { + return spatialTypeResolver.resolveType(); + } + + static class SpatialTypeResolver { + private final SpatialEvaluatorFactory.SpatialSourceResolution supplier; + private final boolean pointsOnly; + + SpatialTypeResolver(SpatialEvaluatorFactory.SpatialSourceResolution supplier, boolean pointsOnly) { + this.supplier = supplier; + this.pointsOnly = pointsOnly; + } + + public Expression left() { + return supplier.left(); + } + + public Expression right() { + return supplier.right(); + } + + public String sourceText() { + return supplier.source().text(); + } + + protected TypeResolution resolveType() { + if (left().foldable() && right().foldable() == false || isNull(left().dataType())) { + // Left is literal, but right is not, check the left field's type against the right field + return resolveType(right(), left(), SECOND, FIRST); + } else { + // All other cases check the right against the left + return resolveType(left(), right(), FIRST, SECOND); + } + } + + protected Expression.TypeResolution isSpatial(Expression e, TypeResolutions.ParamOrdinal paramOrd) { + return pointsOnly + ? EsqlTypeResolutions.isSpatialPoint(e, sourceText(), paramOrd) + : EsqlTypeResolutions.isSpatial(e, sourceText(), paramOrd); + } + + private TypeResolution resolveType( + Expression leftExpression, + Expression rightExpression, + TypeResolutions.ParamOrdinal leftOrdinal, + TypeResolutions.ParamOrdinal rightOrdinal + ) { + TypeResolution leftResolution = isSpatial(leftExpression, leftOrdinal); + TypeResolution rightResolution = isSpatial(rightExpression, rightOrdinal); + if (leftResolution.resolved()) { + return resolveType(leftExpression, rightExpression, rightOrdinal); + } else if (rightResolution.resolved()) { + return resolveType(rightExpression, leftExpression, leftOrdinal); + } else { + return leftResolution; + } + } + + protected TypeResolution resolveType( + Expression spatialExpression, + Expression otherExpression, + TypeResolutions.ParamOrdinal otherParamOrdinal + ) { + if (isNull(spatialExpression.dataType())) { + return isSpatial(otherExpression, otherParamOrdinal); + } + TypeResolution resolution = isSameSpatialType(spatialExpression.dataType(), otherExpression, sourceText(), otherParamOrdinal); + if (resolution.unresolved()) { + return resolution; + } + supplier.setCrsType(spatialExpression.dataType()); + return TypeResolution.TYPE_RESOLVED; + } + + protected TypeResolution isSameSpatialType( + DataType spatialDataType, + Expression expression, + String operationName, + TypeResolutions.ParamOrdinal paramOrd + ) { + return pointsOnly + ? isType(expression, dt -> dt == spatialDataType, operationName, paramOrd, compatibleTypeNames(spatialDataType)) + : isType( + expression, + dt -> EsqlDataTypes.isSpatial(dt) && spatialCRSCompatible(spatialDataType, dt), + operationName, + paramOrd, + compatibleTypeNames(spatialDataType) + ); + } + } + + @Override + public void setCrsType(DataType dataType) { + crsType = SpatialCrsType.fromDataType(dataType); + } + + private static final String[] GEO_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; + private static final String[] CARTESIAN_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; + + protected static boolean spatialCRSCompatible(DataType spatialDataType, DataType otherDataType) { + return EsqlDataTypes.isSpatialGeo(spatialDataType) && EsqlDataTypes.isSpatialGeo(otherDataType) + || EsqlDataTypes.isSpatialGeo(spatialDataType) == false && EsqlDataTypes.isSpatialGeo(otherDataType) == false; + } + + static String[] compatibleTypeNames(DataType spatialDataType) { + return EsqlDataTypes.isSpatialGeo(spatialDataType) ? GEO_TYPE_NAMES : CARTESIAN_TYPE_NAMES; + } + + @Override + public SpatialCrsType crsType() { + if (crsType == null) { + resolveType(); + } + return crsType; + } + + public boolean leftDocValues() { + return leftDocValues; + } + + public boolean rightDocValues() { + return rightDocValues; + } + + /** + * For most spatial functions we only need to know if the CRS is geo or cartesian, not whether the type is point or shape. + * This enum captures this knowledge. + */ + public enum SpatialCrsType { + GEO, + CARTESIAN, + UNSPECIFIED; + + public static SpatialCrsType fromDataType(DataType dataType) { + return EsqlDataTypes.isSpatialGeo(dataType) ? SpatialCrsType.GEO + : EsqlDataTypes.isSpatial(dataType) ? SpatialCrsType.CARTESIAN + : SpatialCrsType.UNSPECIFIED; + } + } + + protected abstract static class BinarySpatialComparator { + protected final SpatialCoordinateTypes spatialCoordinateType; + protected final CoordinateEncoder coordinateEncoder; + protected final SpatialCrsType crsType; + + protected BinarySpatialComparator(SpatialCoordinateTypes spatialCoordinateType, CoordinateEncoder encoder) { + this.spatialCoordinateType = spatialCoordinateType; + this.coordinateEncoder = encoder; + this.crsType = spatialCoordinateType.equals(SpatialCoordinateTypes.GEO) ? SpatialCrsType.GEO : SpatialCrsType.CARTESIAN; + } + + protected Geometry fromBytesRef(BytesRef bytesRef) { + return SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(bytesRef); + } + + protected abstract T compare(BytesRef left, BytesRef right) throws IOException; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java index 14e743c5be460..6fd4f79125a21 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java @@ -79,13 +79,20 @@ interface SpatialSourceSupplier { Expression right(); - SpatialRelatesFunction.SpatialCrsType crsType(); + BinarySpatialFunction.SpatialCrsType crsType(); boolean leftDocValues(); boolean rightDocValues(); } + /** + * When performing type resolution we need also write access to the SpatialSourceSupplier for setting the CRS + */ + interface SpatialSourceResolution extends SpatialSourceSupplier { + void setCrsType(DataType dataType); + } + protected static class SwappedSpatialSourceSupplier implements SpatialSourceSupplier { private final SpatialSourceSupplier delegate; @@ -99,7 +106,7 @@ public Source source() { } @Override - public SpatialRelatesFunction.SpatialCrsType crsType() { + public BinarySpatialFunction.SpatialCrsType crsType() { return delegate.crsType(); } @@ -209,13 +216,13 @@ public EvalOperator.ExpressionEvaluator.Factory get( protected record SpatialEvaluatorFieldKey(DataType dataType, boolean isConstant) {} record SpatialEvaluatorKey( - SpatialRelatesFunction.SpatialCrsType crsType, + BinarySpatialFunction.SpatialCrsType crsType, boolean leftDocValues, boolean rightDocValues, SpatialEvaluatorFieldKey left, SpatialEvaluatorFieldKey right ) { - SpatialEvaluatorKey(SpatialRelatesFunction.SpatialCrsType crsType, SpatialEvaluatorFieldKey left, SpatialEvaluatorFieldKey right) { + SpatialEvaluatorKey(BinarySpatialFunction.SpatialCrsType crsType, SpatialEvaluatorFieldKey left, SpatialEvaluatorFieldKey right) { this(crsType, false, false, left, right); } @@ -229,7 +236,7 @@ SpatialEvaluatorKey swapSides() { static SpatialEvaluatorKey fromSourceAndConstant(DataType left, DataType right) { return new SpatialEvaluatorKey( - SpatialRelatesFunction.SpatialCrsType.fromDataType(left), + BinarySpatialFunction.SpatialCrsType.fromDataType(left), new SpatialEvaluatorFieldKey(left, false), new SpatialEvaluatorFieldKey(right, true) ); @@ -237,7 +244,7 @@ static SpatialEvaluatorKey fromSourceAndConstant(DataType left, DataType right) static SpatialEvaluatorKey fromSources(DataType left, DataType right) { return new SpatialEvaluatorKey( - SpatialRelatesFunction.SpatialCrsType.fromDataType(left), + BinarySpatialFunction.SpatialCrsType.fromDataType(left), new SpatialEvaluatorFieldKey(left, false), new SpatialEvaluatorFieldKey(right, false) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index 064df31e35cb2..ca9b6838cc2ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -19,8 +19,6 @@ import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; -import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; @@ -36,28 +34,16 @@ import static org.apache.lucene.document.ShapeField.QueryRelation.CONTAINS; import static org.apache.lucene.document.ShapeField.QueryRelation.DISJOINT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; -import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; -import static org.elasticsearch.xpack.esql.core.type.DataType.isNull; -import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D; -public abstract class SpatialRelatesFunction extends BinaryScalarFunction +public abstract class SpatialRelatesFunction extends BinarySpatialFunction implements EvaluatorMapper, SpatialEvaluatorFactory.SpatialSourceSupplier { - protected SpatialCrsType crsType; - protected final boolean leftDocValues; - protected final boolean rightDocValues; protected SpatialRelatesFunction(Source source, Expression left, Expression right, boolean leftDocValues, boolean rightDocValues) { - super(source, left, right); - this.leftDocValues = leftDocValues; - this.rightDocValues = rightDocValues; + super(source, left, right, leftDocValues, rightDocValues, false); } public abstract ShapeField.QueryRelation queryRelation(); @@ -67,94 +53,6 @@ public DataType dataType() { return DataType.BOOLEAN; } - @Override - public SpatialCrsType crsType() { - if (crsType == null) { - resolveType(); - } - return crsType; - } - - @Override - protected TypeResolution resolveType() { - if (left().foldable() && right().foldable() == false || isNull(left().dataType())) { - // Left is literal, but right is not, check the left field's type against the right field - return resolveType(right(), left(), SECOND, FIRST); - } else { - // All other cases check the right against the left - return resolveType(left(), right(), FIRST, SECOND); - } - } - - private TypeResolution resolveType( - Expression leftExpression, - Expression rightExpression, - TypeResolutions.ParamOrdinal leftOrdinal, - TypeResolutions.ParamOrdinal rightOrdinal - ) { - TypeResolution leftResolution = isSpatial(leftExpression, sourceText(), leftOrdinal); - TypeResolution rightResolution = isSpatial(rightExpression, sourceText(), rightOrdinal); - if (leftResolution.resolved()) { - return resolveType(leftExpression, rightExpression, rightOrdinal); - } else if (rightResolution.resolved()) { - return resolveType(rightExpression, leftExpression, leftOrdinal); - } else { - return leftResolution; - } - } - - protected TypeResolution resolveType( - Expression spatialExpression, - Expression otherExpression, - TypeResolutions.ParamOrdinal otherParamOrdinal - ) { - if (isNull(spatialExpression.dataType())) { - return isSpatial(otherExpression, sourceText(), otherParamOrdinal); - } - TypeResolution resolution = isSameSpatialType(spatialExpression.dataType(), otherExpression, sourceText(), otherParamOrdinal); - if (resolution.unresolved()) { - return resolution; - } - setCrsType(spatialExpression.dataType()); - return TypeResolution.TYPE_RESOLVED; - } - - protected void setCrsType(DataType dataType) { - crsType = SpatialCrsType.fromDataType(dataType); - } - - public static TypeResolution isSameSpatialType( - DataType spatialDataType, - Expression expression, - String operationName, - TypeResolutions.ParamOrdinal paramOrd - ) { - return isType( - expression, - dt -> EsqlDataTypes.isSpatial(dt) && spatialCRSCompatible(spatialDataType, dt), - operationName, - paramOrd, - compatibleTypeNames(spatialDataType) - ); - } - - private static final String[] GEO_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; - private static final String[] CARTESIAN_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() }; - - private static boolean spatialCRSCompatible(DataType spatialDataType, DataType otherDataType) { - return EsqlDataTypes.isSpatialGeo(spatialDataType) && EsqlDataTypes.isSpatialGeo(otherDataType) - || EsqlDataTypes.isSpatialGeo(spatialDataType) == false && EsqlDataTypes.isSpatialGeo(otherDataType) == false; - } - - static String[] compatibleTypeNames(DataType spatialDataType) { - return EsqlDataTypes.isSpatialGeo(spatialDataType) ? GEO_TYPE_NAMES : CARTESIAN_TYPE_NAMES; - } - - @Override - public boolean foldable() { - return left().foldable() && right().foldable(); - } - /** * Mark the function as expecting the specified fields to arrive as doc-values. */ @@ -196,14 +94,6 @@ public boolean equals(Object obj) { return false; } - public boolean leftDocValues() { - return leftDocValues; - } - - public boolean rightDocValues() { - return rightDocValues; - } - /** * Produce a map of rules defining combinations of incoming types to the evaluator factory that should be used. */ @@ -236,24 +126,9 @@ protected boolean foundField(Expression expression, Set foundAtt return expression instanceof FieldAttribute field && foundAttributes.contains(field); } - protected enum SpatialCrsType { - GEO, - CARTESIAN, - UNSPECIFIED; - - public static SpatialCrsType fromDataType(DataType dataType) { - return EsqlDataTypes.isSpatialGeo(dataType) ? SpatialCrsType.GEO - : EsqlDataTypes.isSpatial(dataType) ? SpatialCrsType.CARTESIAN - : SpatialCrsType.UNSPECIFIED; - } - } - - protected static class SpatialRelations { + protected static class SpatialRelations extends BinarySpatialComparator { protected final ShapeField.QueryRelation queryRelation; - protected final SpatialCoordinateTypes spatialCoordinateType; - protected final CoordinateEncoder coordinateEncoder; protected final ShapeIndexer shapeIndexer; - protected final SpatialCrsType crsType; protected SpatialRelations( ShapeField.QueryRelation queryRelation, @@ -261,11 +136,14 @@ protected SpatialRelations( CoordinateEncoder encoder, ShapeIndexer shapeIndexer ) { + super(spatialCoordinateType, encoder); this.queryRelation = queryRelation; - this.spatialCoordinateType = spatialCoordinateType; - this.coordinateEncoder = encoder; this.shapeIndexer = shapeIndexer; - this.crsType = spatialCoordinateType.equals(SpatialCoordinateTypes.GEO) ? SpatialCrsType.GEO : SpatialCrsType.CARTESIAN; + } + + @Override + protected Boolean compare(BytesRef left, BytesRef right) throws IOException { + return geometryRelatesGeometry(left, right); } protected boolean geometryRelatesGeometry(BytesRef left, BytesRef right) throws IOException { @@ -273,10 +151,6 @@ protected boolean geometryRelatesGeometry(BytesRef left, BytesRef right) throws return geometryRelatesGeometry(left, rightComponent2D); } - protected Geometry fromBytesRef(BytesRef bytesRef) { - return SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(bytesRef); - } - protected boolean geometryRelatesGeometry(BytesRef left, Component2D rightComponent2D) throws IOException { Geometry leftGeom = fromBytesRef(left); // We already have a Component2D for the right geometry, so we need to convert the left geometry to a doc-values byte array diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java index 3278eaac43d0d..600c3529acd13 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java @@ -36,12 +36,12 @@ public class SpatialRelatesUtils { * This function is used to convert a spatial constant to a lucene Component2D. * When both left and right sides are constants, we convert the left to a doc-values byte array and the right to a Component2D. */ - static Component2D asLuceneComponent2D(SpatialRelatesFunction.SpatialCrsType crsType, Expression expression) { + static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { return asLuceneComponent2D(crsType, makeGeometryFromLiteral(expression)); } - static Component2D asLuceneComponent2D(SpatialRelatesFunction.SpatialCrsType crsType, Geometry geometry) { - if (crsType == SpatialRelatesFunction.SpatialCrsType.GEO) { + static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType crsType, Geometry geometry) { + if (crsType == BinarySpatialFunction.SpatialCrsType.GEO) { var luceneGeometries = LuceneGeometriesUtils.toLatLonGeometry(geometry, true, t -> {}); return LatLonGeometry.create(luceneGeometries); } else { @@ -55,12 +55,12 @@ static Component2D asLuceneComponent2D(SpatialRelatesFunction.SpatialCrsType crs * When both left and right sides are constants, we convert the left to a doc-values byte array and the right to a Component2D[]. * The reason for generating an array instead of a single component is for multi-shape support with ST_CONTAINS. */ - static Component2D[] asLuceneComponent2Ds(SpatialRelatesFunction.SpatialCrsType crsType, Expression expression) { + static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(expression)); } - static Component2D[] asLuceneComponent2Ds(SpatialRelatesFunction.SpatialCrsType crsType, Geometry geometry) { - if (crsType == SpatialRelatesFunction.SpatialCrsType.GEO) { + static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType crsType, Geometry geometry) { + if (crsType == BinarySpatialFunction.SpatialCrsType.GEO) { var luceneGeometries = LuceneGeometriesUtils.toLatLonGeometry(geometry, true, t -> {}); return LuceneComponent2DUtils.createLatLonComponents(luceneGeometries); } else { @@ -73,10 +73,10 @@ static Component2D[] asLuceneComponent2Ds(SpatialRelatesFunction.SpatialCrsType * This function is used to convert a spatial constant to a doc-values byte array. * When both left and right sides are constants, we convert the left to a doc-values byte array and the right to a Component2D. */ - static GeometryDocValueReader asGeometryDocValueReader(SpatialRelatesFunction.SpatialCrsType crsType, Expression expression) + static GeometryDocValueReader asGeometryDocValueReader(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) throws IOException { Geometry geometry = makeGeometryFromLiteral(expression); - if (crsType == SpatialRelatesFunction.SpatialCrsType.GEO) { + if (crsType == BinarySpatialFunction.SpatialCrsType.GEO) { return asGeometryDocValueReader( CoordinateEncoder.GEO, new GeoShapeIndexer(Orientation.CCW, "SpatialRelatesFunction"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java new file mode 100644 index 0000000000000..89c048f7eace8 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SloppyMath; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.makeGeometryFromLiteral; + +/** + * Computes the distance between two points. + * For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. + * For geographic geometries, this is the circular distance along the great circle in meters. + * The function `st_distance` is defined in the OGC Simple Feature Access standard. + * Alternatively it is described in PostGIS documentation at PostGIS:ST_Distance. + */ +public class StDistance extends BinarySpatialFunction implements EvaluatorMapper { + // public for test access with reflection + public static final DistanceCalculator GEO = new GeoDistanceCalculator(); + // public for test access with reflection + public static final DistanceCalculator CARTESIAN = new CartesianDistanceCalculator(); + + protected static class GeoDistanceCalculator extends DistanceCalculator { + protected GeoDistanceCalculator() { + super(SpatialCoordinateTypes.GEO, CoordinateEncoder.GEO); + } + + @Override + protected double distance(Point left, Point right) { + // TODO: investigate if we need to use the more complex validation in Lucenes Circle2D::HaversinDistance class + return SloppyMath.haversinMeters(left.getY(), left.getX(), right.getY(), right.getX()); + } + } + + protected static class CartesianDistanceCalculator extends DistanceCalculator { + + protected CartesianDistanceCalculator() { + super(SpatialCoordinateTypes.CARTESIAN, CoordinateEncoder.CARTESIAN); + } + + @Override + protected double distance(Point left, Point right) { + final double diffX = left.getX() - right.getX(); + final double diffY = left.getY() - right.getY(); + return Math.sqrt(diffX * diffX + diffY * diffY); + } + } + + /** + * This class is a CRS specific interface for generalizing distance calculations for the various possible ways + * that the geometries can be provided, from source, from evals, from literals and from doc values. + */ + public abstract static class DistanceCalculator extends BinarySpatialComparator { + + protected DistanceCalculator(SpatialCoordinateTypes spatialCoordinateType, CoordinateEncoder encoder) { + super(spatialCoordinateType, encoder); + } + + @Override + protected Double compare(BytesRef left, BytesRef right) throws IOException { + return distance(left, right); + } + + protected abstract double distance(Point left, Point right); + + protected double distance(long encoded, Geometry right) { + Point point = spatialCoordinateType.longAsPoint(encoded); + return distance(point, (Point) right); + } + + protected double distance(Geometry left, Geometry right) { + return distance((Point) left, (Point) right); + } + + public double distance(BytesRef left, BytesRef right) { + return distance(this.fromBytesRef(left), this.fromBytesRef(right)); + } + + public double distance(BytesRef left, Point right) { + return distance(this.fromBytesRef(left), right); + } + } + + @FunctionInfo( + returnType = "double", + description = """ + Computes the distance between two points. + For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. + For geographic geometries, this is the circular distance along the great circle in meters.""", + examples = @Example(file = "spatial", tag = "st_distance-airports") + ) + public StDistance( + Source source, + @Param(name = "geomA", type = { "geo_point", "cartesian_point" }, description = """ + Expression of type `geo_point` or `cartesian_point`. + If `null`, the function returns `null`.""") Expression left, + @Param(name = "geomB", type = { "geo_point", "cartesian_point" }, description = """ + Expression of type `geo_point` or `cartesian_point`. + If `null`, the function returns `null`. + The second parameter must also have the same coordinate system as the first. + This means it is not possible to combine `geo_point` and `cartesian_point` parameters.""") Expression right + ) { + super(source, left, right, false, false, true); + } + + protected StDistance(Source source, Expression left, Expression right, boolean leftDocValues, boolean rightDocValues) { + super(source, left, right, leftDocValues, rightDocValues, true); + } + + @Override + public DataType dataType() { + return DOUBLE; + } + + @Override + protected StDistance replaceChildren(Expression newLeft, Expression newRight) { + return new StDistance(source(), newLeft, newRight, leftDocValues, rightDocValues); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StDistance::new, left(), right()); + } + + @Override + public Object fold() { + var leftGeom = makeGeometryFromLiteral(left()); + var rightGeom = makeGeometryFromLiteral(right()); + return (crsType == SpatialCrsType.GEO) ? GEO.distance(leftGeom, rightGeom) : CARTESIAN.distance(leftGeom, rightGeom); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator + ) { + if (right().foldable()) { + return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(right()), leftDocValues); + } else if (left().foldable()) { + return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(left()), rightDocValues); + } else { + EvalOperator.ExpressionEvaluator.Factory leftE = toEvaluator.apply(left()); + EvalOperator.ExpressionEvaluator.Factory rightE = toEvaluator.apply(right()); + if (crsType() == SpatialCrsType.GEO) { + if (leftDocValues) { + return new StDistanceGeoPointDocValuesAndSourceEvaluator.Factory(source(), leftE, rightE); + } else if (rightDocValues) { + return new StDistanceGeoPointDocValuesAndSourceEvaluator.Factory(source(), rightE, leftE); + } else { + return new StDistanceGeoSourceAndSourceEvaluator.Factory(source(), leftE, rightE); + } + } else if (crsType() == SpatialCrsType.CARTESIAN) { + if (leftDocValues) { + return new StDistanceCartesianPointDocValuesAndSourceEvaluator.Factory(source(), leftE, rightE); + } else if (rightDocValues) { + return new StDistanceCartesianPointDocValuesAndSourceEvaluator.Factory(source(), rightE, leftE); + } else { + return new StDistanceCartesianSourceAndSourceEvaluator.Factory(source(), leftE, rightE); + } + } + } + throw EsqlIllegalArgumentException.illegalDataType(crsType().name()); + } + + private EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator, + Expression field, + Geometry geometry, + boolean docValues + ) { + if (geometry instanceof Point point) { + return toEvaluator(toEvaluator, field, point, docValues); + } else { + throw new IllegalArgumentException("Unsupported geometry type for ST_DISTANCE: " + geometry.type().name()); + } + } + + private EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator, + Expression field, + Point point, + boolean docValues + ) { + EvalOperator.ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field); + if (crsType() == SpatialCrsType.GEO) { + if (docValues) { + return new StDistanceGeoPointDocValuesAndConstantEvaluator.Factory(source(), fieldEvaluator, point); + } else { + return new StDistanceGeoSourceAndConstantEvaluator.Factory(source(), fieldEvaluator, point); + } + } else if (crsType() == SpatialCrsType.CARTESIAN) { + if (docValues) { + return new StDistanceCartesianPointDocValuesAndConstantEvaluator.Factory(source(), fieldEvaluator, point); + } else { + return new StDistanceCartesianSourceAndConstantEvaluator.Factory(source(), fieldEvaluator, point); + } + } + throw EsqlIllegalArgumentException.illegalDataType(crsType().name()); + } + + @Evaluator(extraName = "GeoSourceAndConstant", warnExceptions = { IllegalArgumentException.class, IOException.class }) + static double processGeoSourceAndConstant(BytesRef leftValue, @Fixed Point rightValue) throws IOException { + return GEO.distance(leftValue, rightValue); + } + + @Evaluator(extraName = "GeoSourceAndSource", warnExceptions = { IllegalArgumentException.class, IOException.class }) + static double processGeoSourceAndSource(BytesRef leftValue, BytesRef rightValue) throws IOException { + return GEO.distance(leftValue, rightValue); + } + + @Evaluator(extraName = "GeoPointDocValuesAndConstant", warnExceptions = { IllegalArgumentException.class }) + static double processGeoPointDocValuesAndConstant(long leftValue, @Fixed Point rightValue) { + return GEO.distance(leftValue, rightValue); + } + + @Evaluator(extraName = "GeoPointDocValuesAndSource", warnExceptions = { IllegalArgumentException.class }) + static double processGeoPointDocValuesAndSource(long leftValue, BytesRef rightValue) { + Geometry geometry = SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(rightValue); + return GEO.distance(leftValue, geometry); + } + + @Evaluator(extraName = "CartesianSourceAndConstant", warnExceptions = { IllegalArgumentException.class, IOException.class }) + static double processCartesianSourceAndConstant(BytesRef leftValue, @Fixed Point rightValue) throws IOException { + return CARTESIAN.distance(leftValue, rightValue); + } + + @Evaluator(extraName = "CartesianSourceAndSource", warnExceptions = { IllegalArgumentException.class, IOException.class }) + static double processCartesianSourceAndSource(BytesRef leftValue, BytesRef rightValue) throws IOException { + return CARTESIAN.distance(leftValue, rightValue); + } + + @Evaluator(extraName = "CartesianPointDocValuesAndConstant", warnExceptions = { IllegalArgumentException.class }) + static double processCartesianPointDocValuesAndConstant(long leftValue, @Fixed Point rightValue) { + return CARTESIAN.distance(leftValue, rightValue); + } + + @Evaluator(extraName = "CartesianPointDocValuesAndSource") + static double processCartesianPointDocValuesAndSource(long leftValue, BytesRef rightValue) { + Geometry geometry = SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(rightValue); + return CARTESIAN.distance(leftValue, geometry); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 74e8661596e41..da96c227fff43 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -74,11 +74,12 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Round; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialContains; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialDisjoint; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; -import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; @@ -244,10 +245,11 @@ public static List namedTypeEntries() { of(ScalarFunction.class, Pow.class, PlanNamedTypes::writePow, PlanNamedTypes::readPow), of(ScalarFunction.class, StartsWith.class, PlanNamedTypes::writeStartsWith, PlanNamedTypes::readStartsWith), of(ScalarFunction.class, EndsWith.class, PlanNamedTypes::writeEndsWith, PlanNamedTypes::readEndsWith), - of(ScalarFunction.class, SpatialIntersects.class, PlanNamedTypes::writeSpatialRelatesFunction, PlanNamedTypes::readIntersects), - of(ScalarFunction.class, SpatialDisjoint.class, PlanNamedTypes::writeSpatialRelatesFunction, PlanNamedTypes::readDisjoint), - of(ScalarFunction.class, SpatialContains.class, PlanNamedTypes::writeSpatialRelatesFunction, PlanNamedTypes::readContains), - of(ScalarFunction.class, SpatialWithin.class, PlanNamedTypes::writeSpatialRelatesFunction, PlanNamedTypes::readWithin), + of(ScalarFunction.class, SpatialIntersects.class, PlanNamedTypes::writeBinarySpatialFunction, PlanNamedTypes::readIntersects), + of(ScalarFunction.class, SpatialDisjoint.class, PlanNamedTypes::writeBinarySpatialFunction, PlanNamedTypes::readDisjoint), + of(ScalarFunction.class, SpatialContains.class, PlanNamedTypes::writeBinarySpatialFunction, PlanNamedTypes::readContains), + of(ScalarFunction.class, SpatialWithin.class, PlanNamedTypes::writeBinarySpatialFunction, PlanNamedTypes::readWithin), + of(ScalarFunction.class, StDistance.class, PlanNamedTypes::writeBinarySpatialFunction, PlanNamedTypes::readDistance), of(ScalarFunction.class, Substring.class, PlanNamedTypes::writeSubstring, PlanNamedTypes::readSubstring), of(ScalarFunction.class, Locate.class, PlanNamedTypes::writeLocate, PlanNamedTypes::readLocate), of(ScalarFunction.class, Left.class, PlanNamedTypes::writeLeft, PlanNamedTypes::readLeft), @@ -1080,9 +1082,13 @@ static SpatialWithin readWithin(PlanStreamInput in) throws IOException { return new SpatialWithin(Source.EMPTY, in.readExpression(), in.readExpression()); } - static void writeSpatialRelatesFunction(PlanStreamOutput out, SpatialRelatesFunction spatialRelatesFunction) throws IOException { - out.writeExpression(spatialRelatesFunction.left()); - out.writeExpression(spatialRelatesFunction.right()); + static StDistance readDistance(PlanStreamInput in) throws IOException { + return new StDistance(Source.EMPTY, in.readExpression(), in.readExpression()); + } + + static void writeBinarySpatialFunction(PlanStreamOutput out, BinarySpatialFunction binarySpatialFunction) throws IOException { + out.writeExpression(binarySpatialFunction.left()); + out.writeExpression(binarySpatialFunction.right()); } static Round readRound(PlanStreamInput in) throws IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index dc7e09dc8f174..b5700d38c2d10 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -345,17 +345,17 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte List inverse = source.layout.inverse(); for (int channel = 0; channel < inverse.size(); channel++) { elementTypes[channel] = PlannerUtils.toElementType(inverse.get(channel).type()); - encoders[channel] = switch (inverse.get(channel).type().typeName()) { - case "ip" -> TopNEncoder.IP; - case "text", "keyword" -> TopNEncoder.UTF8; - case "version" -> TopNEncoder.VERSION; - case "boolean", "null", "byte", "short", "integer", "long", "double", "float", "half_float", "datetime", "date_period", - "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc", "_tsid" -> TopNEncoder.DEFAULT_SORTABLE; - case "geo_point", "cartesian_point", "geo_shape", "cartesian_shape", "counter_long", "counter_integer", "counter_double" -> + encoders[channel] = switch (inverse.get(channel).type()) { + case IP -> TopNEncoder.IP; + case TEXT, KEYWORD -> TopNEncoder.UTF8; + case VERSION -> TopNEncoder.VERSION; + case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_PERIOD, TIME_DURATION, OBJECT, + NESTED, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE; + case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE -> TopNEncoder.DEFAULT_UNSORTABLE; // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point - case "unsupported" -> TopNEncoder.UNSUPPORTED; - default -> throw new EsqlIllegalArgumentException("No TopN sorting encoder for type " + inverse.get(channel).type()); + case UNSUPPORTED -> TopNEncoder.UNSUPPORTED; + case SOURCE -> throw new EsqlIllegalArgumentException("No TopN sorting encoder for type " + inverse.get(channel).type()); }; } List orders = topNExec.order().stream().map(order -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 6545e892741d2..5d5e2b82e4b7b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -223,6 +223,7 @@ Stats stats() { private static class LuceneQuery extends org.apache.lucene.search.Query { final org.apache.lucene.search.Query next; private final IndexFieldData fieldData; + // mutable object for collecting stats and warnings, not really part of the query private final Stats stats; private final Warnings warnings; @@ -267,14 +268,12 @@ public boolean equals(Object obj) { return false; } SingleValueQuery.LuceneQuery other = (SingleValueQuery.LuceneQuery) obj; - return next.equals(other.next) - && fieldData.getFieldName().equals(other.fieldData.getFieldName()) - && warnings.equals(other.warnings); + return next.equals(other.next) && fieldData.getFieldName().equals(other.fieldData.getFieldName()); } @Override public int hashCode() { - return Objects.hash(classHash(), next, fieldData, warnings); + return Objects.hash(classHash(), next, fieldData.getFieldName()); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index 42022099ceace..83f5a621c93a5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -105,8 +105,8 @@ public static Iterable parameters() { protected static void addSpatialCombinations(List suppliers) { for (DataType dataType : List.of(DataType.GEO_POINT, DataType.GEO_SHAPE, DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE)) { - TestCaseSupplier.TypedDataSupplier leftDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType); - TestCaseSupplier.TypedDataSupplier rightDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType); + TestCaseSupplier.TypedDataSupplier leftDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType, false); + TestCaseSupplier.TypedDataSupplier rightDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType, false); suppliers.add( TestCaseSupplier.testCaseSupplier( leftDataSupplier, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java new file mode 100644 index 0000000000000..37e09caf0d105 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java @@ -0,0 +1,293 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import joptsimple.internal.Strings; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.function.BinaryOperator; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction.compatibleTypeNames; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatialGeo; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isString; +import static org.hamcrest.Matchers.equalTo; + +public abstract class BinarySpatialFunctionTestCase extends AbstractFunctionTestCase { + + private static String getFunctionClassName() { + Class testClass = getTestClass(); + String testClassName = testClass.getSimpleName(); + return testClassName.replace("Tests", ""); + } + + protected static Class getSpatialRelatesFunctionClass() throws ClassNotFoundException { + String functionClassName = getFunctionClassName(); + return Class.forName("org.elasticsearch.xpack.esql.expression.function.scalar.spatial." + functionClassName); + } + + public static TestCaseSupplier.TypedDataSupplier testCaseSupplier(DataType dataType, boolean pointsOnly) { + if (pointsOnly) { + return switch (dataType.esType()) { + case "geo_point" -> TestCaseSupplier.geoPointCases(() -> false).get(0); + case "cartesian_point" -> TestCaseSupplier.cartesianPointCases(() -> false).get(0); + default -> throw new IllegalArgumentException("Unsupported datatype for " + functionName() + ": " + dataType); + }; + } else { + return switch (dataType.esType()) { + case "geo_point" -> TestCaseSupplier.geoPointCases(() -> false).get(0); + case "geo_shape" -> TestCaseSupplier.geoShapeCases(() -> false).get(0); + case "cartesian_point" -> TestCaseSupplier.cartesianPointCases(() -> false).get(0); + case "cartesian_shape" -> TestCaseSupplier.cartesianShapeCases(() -> false).get(0); + default -> throw new IllegalArgumentException("Unsupported datatype for " + functionName() + ": " + dataType); + }; + } + } + + /** + * Binary spatial functions that take two spatial arguments + * should use this to generate combinations of test cases. + */ + protected static void addSpatialCombinations( + List suppliers, + DataType[] dataTypes, + DataType returnType, + boolean pointsOnly + ) { + for (DataType leftType : dataTypes) { + TestCaseSupplier.TypedDataSupplier leftDataSupplier = testCaseSupplier(leftType, pointsOnly); + for (DataType rightType : dataTypes) { + if (typeCompatible(leftType, rightType)) { + TestCaseSupplier.TypedDataSupplier rightDataSupplier = testCaseSupplier(rightType, pointsOnly); + suppliers.add( + TestCaseSupplier.testCaseSupplier( + leftDataSupplier, + rightDataSupplier, + BinarySpatialFunctionTestCase::spatialEvaluatorString, + returnType, + (l, r) -> expected(l, leftType, r, rightType) + ) + ); + } + } + } + } + + /** + * Build the expected error message for an invalid type signature. + * For two args, this assumes they are both spatial. + * For three args, we assume two spatial and one additional numerical argument, treated differently. + */ + protected static String typeErrorMessage( + boolean includeOrdinal, + List> validPerPosition, + List types, + boolean pointsOnly + ) { + boolean argInvalid = false; + List badArgPositions = new ArrayList<>(); + for (int i = 0; i < types.size(); i++) { + if (validPerPosition.get(i).contains(types.get(i)) == false) { + if (i == 2) { + argInvalid = true; + } else { + badArgPositions.add(i); + } + } + } + if (badArgPositions.isEmpty() && types.get(0) != DataType.NULL && types.get(1) != DataType.NULL) { + // First two arguments are valid spatial types, but it is still possible they are incompatible + var leftCrs = BinarySpatialFunction.SpatialCrsType.fromDataType(types.get(0)); + var rightCrs = BinarySpatialFunction.SpatialCrsType.fromDataType(types.get(1)); + if (leftCrs != rightCrs) { + badArgPositions.add(1); + } + } + if (badArgPositions.size() == 1) { + int badArgPosition = badArgPositions.get(0); + int goodArgPosition = badArgPosition == 0 ? 1 : 0; + if (isSpatial(types.get(goodArgPosition)) == false) { + return oneInvalid(badArgPosition, -1, includeOrdinal, types, pointsOnly); + } else { + return oneInvalid(badArgPosition, goodArgPosition, includeOrdinal, types, pointsOnly); + } + } else if (argInvalid && badArgPositions.size() != 2) { + return invalidArg(types.get(2)); + } else { + return oneInvalid(0, -1, includeOrdinal, types, pointsOnly); + } + } + + private static String invalidArg(DataType invalidType) { + return String.format( + Locale.ROOT, + "%s argument of [%s] must be [%s], found value [%s] type [%s]", + TypeResolutions.ParamOrdinal.fromIndex(2).toString().toLowerCase(Locale.ROOT), + "", + "double", + invalidType.typeName(), + invalidType.typeName() + ); + } + + private static String oneInvalid( + int badArgPosition, + int goodArgPosition, + boolean includeOrdinal, + List types, + boolean pointsOnly + ) { + String expected = pointsOnly ? "geo_point or cartesian_point" : "geo_point, cartesian_point, geo_shape or cartesian_shape"; + String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; + String expectedType = goodArgPosition >= 0 ? compatibleTypes(types.get(goodArgPosition)) : expected; + String name = types.get(badArgPosition).typeName(); + return ordinal + "argument of [] must be [" + expectedType + "], found value [" + name + "] type [" + name + "]"; + } + + private static String compatibleTypes(DataType spatialDataType) { + return Strings.join(compatibleTypeNames(spatialDataType), " or "); + } + + protected static Object expected(Object left, DataType leftType, Object right, DataType rightType) { + if (typeCompatible(leftType, rightType) == false) { + return null; + } + // TODO cast objects to right type and check intersection + BytesRef leftWKB = asGeometryWKB(left, leftType); + BytesRef rightWKB = asGeometryWKB(right, rightType); + BinarySpatialFunction.BinarySpatialComparator spatialRelations = spatialRelations(left, leftType, right, rightType); + try { + return spatialRelations.compare(leftWKB, rightWKB); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * When two spatial arguments are processed and then compared with a third argument, + * we need to process this argument too, before producing the final result. + */ + protected static Object expected( + Object left, + DataType leftType, + Object right, + DataType rightType, + Object arg, + BinaryOperator argProcessor + ) { + Object result = expected(left, leftType, right, rightType); + if (result == null) { + return null; + } + return argProcessor.apply(result, arg); + } + + private static BinarySpatialFunction.BinarySpatialComparator getRelationsField(String name) { + try { + Field field = getSpatialRelatesFunctionClass().getField(name); + Object value = field.get(null); + return (BinarySpatialFunction.BinarySpatialComparator) value; + } catch (NoSuchFieldException | ClassNotFoundException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static BinarySpatialFunction.BinarySpatialComparator spatialRelations( + Object left, + DataType leftType, + Object right, + DataType rightType + ) { + if (isSpatialGeo(leftType) || isSpatialGeo(rightType)) { + return getRelationsField("GEO"); + } else if (isSpatial(leftType) || isSpatial(rightType)) { + return getRelationsField("CARTESIAN"); + } else { + throw new IllegalArgumentException( + "Unsupported left and right types: left[" + + leftType.esType() + + ":" + + left.getClass().getSimpleName() + + "] right[" + + rightType.esType() + + ":" + + right.getClass().getSimpleName() + + "]" + ); + } + } + + protected static BytesRef asGeometryWKB(Object object, DataType dataType) { + if (isString(dataType)) { + return SpatialCoordinateTypes.UNSPECIFIED.wktToWkb(object.toString()); + } else if (object instanceof BytesRef wkb) { + return wkb; + } else { + throw new IllegalArgumentException("Invalid geometry base type for " + dataType + ": " + object.getClass().getSimpleName()); + } + } + + protected static boolean typeCompatible(DataType leftType, DataType rightType) { + if (isSpatial(leftType) && isSpatial(rightType)) { + // Both must be GEO_* or both must be CARTESIAN_* + return countGeo(leftType, rightType) != 1; + } + return true; + } + + private static DataType pickSpatialType(DataType leftType, DataType rightType) { + if (isSpatial(leftType)) { + return leftType; + } else if (isSpatial(rightType)) { + return rightType; + } else { + throw new IllegalArgumentException("Invalid spatial types: " + leftType + " and " + rightType); + } + } + + private static Matcher spatialEvaluatorString(DataType leftType, DataType rightType) { + String crsType = isSpatialGeo(pickSpatialType(leftType, rightType)) ? "Geo" : "Cartesian"; + String channels = channelsText("leftValue", "rightValue"); + return equalTo(getFunctionClassName() + crsType + "SourceAndSourceEvaluator[" + channels + "]"); + } + + private static Matcher spatialEvaluatorString(DataType leftType, DataType rightType, DataType argType) { + String crsType = isSpatialGeo(pickSpatialType(leftType, rightType)) ? "Geo" : "Cartesian"; + String channels = channelsText("leftValue", "rightValue", "argValue"); + return equalTo(getFunctionClassName() + crsType + "FieldAndFieldAndFieldEvaluator[" + channels + "]"); + } + + private static String channelsText(String... args) { + return IntStream.range(0, args.length).mapToObj(i -> args[i] + "=Attribute[channel=" + i + "]").collect(Collectors.joining(", ")); + } + + private static int countGeo(DataType... types) { + int count = 0; + for (DataType type : types) { + if (isSpatialGeo(type)) { + count++; + } + } + return count; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java index 9929971c48613..53ed472a4d43f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java @@ -7,202 +7,19 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; -import joptsimple.internal.Strings; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.hamcrest.Matcher; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Set; -import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction.compatibleTypeNames; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatialGeo; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isString; -import static org.hamcrest.Matchers.equalTo; - -public abstract class SpatialRelatesFunctionTestCase extends AbstractFunctionTestCase { - - private static String getFunctionClassName() { - Class testClass = getTestClass(); - String testClassName = testClass.getSimpleName(); - return testClassName.replace("Tests", ""); - } - - private static Class getSpatialRelatesFunctionClass() throws ClassNotFoundException { - String functionClassName = getFunctionClassName(); - return Class.forName("org.elasticsearch.xpack.esql.expression.function.scalar.spatial." + functionClassName); - } - - private static SpatialRelatesFunction.SpatialRelations getRelationsField(String name) { - try { - Field field = getSpatialRelatesFunctionClass().getField(name); - Object value = field.get(null); - return (SpatialRelatesFunction.SpatialRelations) value; - } catch (NoSuchFieldException | ClassNotFoundException | IllegalAccessException e) { - throw new RuntimeException(e); - } - } +public abstract class SpatialRelatesFunctionTestCase extends BinarySpatialFunctionTestCase { protected static void addSpatialCombinations(List suppliers, DataType[] dataTypes) { - for (DataType leftType : dataTypes) { - TestCaseSupplier.TypedDataSupplier leftDataSupplier = testCaseSupplier(leftType); - for (DataType rightType : dataTypes) { - if (typeCompatible(leftType, rightType)) { - TestCaseSupplier.TypedDataSupplier rightDataSupplier = testCaseSupplier(rightType); - suppliers.add( - TestCaseSupplier.testCaseSupplier( - leftDataSupplier, - rightDataSupplier, - SpatialRelatesFunctionTestCase::spatialEvaluatorString, - DataType.BOOLEAN, - (l, r) -> expected(l, leftType, r, rightType) - ) - ); - } - } - } + addSpatialCombinations(suppliers, dataTypes, DataType.BOOLEAN, false); } - /** - * Build the expected error message for an invalid type signature. - */ protected static String typeErrorMessage(boolean includeOrdinal, List> validPerPosition, List types) { - List badArgPositions = new ArrayList<>(); - for (int i = 0; i < types.size(); i++) { - if (validPerPosition.get(i).contains(types.get(i)) == false) { - badArgPositions.add(i); - } - } - if (badArgPositions.isEmpty()) { - return oneInvalid(1, 0, includeOrdinal, types); - } else if (badArgPositions.size() == 1) { - int badArgPosition = badArgPositions.get(0); - int goodArgPosition = badArgPosition == 0 ? 1 : 0; - if (isSpatial(types.get(goodArgPosition)) == false) { - return oneInvalid(badArgPosition, -1, includeOrdinal, types); - } else { - return oneInvalid(badArgPosition, goodArgPosition, includeOrdinal, types); - } - } else { - return oneInvalid(0, -1, includeOrdinal, types); - } - } - - private static String oneInvalid(int badArgPosition, int goodArgPosition, boolean includeOrdinal, List types) { - String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; - String expectedType = goodArgPosition >= 0 - ? compatibleTypes(types.get(goodArgPosition)) - : "geo_point, cartesian_point, geo_shape or cartesian_shape"; - String name = types.get(badArgPosition).typeName(); - return ordinal + "argument of [] must be [" + expectedType + "], found value [" + name + "] type [" + name + "]"; - } - - private static String compatibleTypes(DataType spatialDataType) { - return Strings.join(compatibleTypeNames(spatialDataType), " or "); - } - - public static TestCaseSupplier.TypedDataSupplier testCaseSupplier(DataType dataType) { - return switch (dataType.esType()) { - case "geo_point" -> TestCaseSupplier.geoPointCases(() -> false).get(0); - case "geo_shape" -> TestCaseSupplier.geoShapeCases(() -> false).get(0); - case "cartesian_point" -> TestCaseSupplier.cartesianPointCases(() -> false).get(0); - case "cartesian_shape" -> TestCaseSupplier.cartesianShapeCases(() -> false).get(0); - default -> throw new IllegalArgumentException("Unsupported datatype for " + functionName() + ": " + dataType); - }; - } - - private static Object expected(Object left, DataType leftType, Object right, DataType rightType) { - if (typeCompatible(leftType, rightType) == false) { - return null; - } - // TODO cast objects to right type and check intersection - BytesRef leftWKB = asGeometryWKB(left, leftType); - BytesRef rightWKB = asGeometryWKB(right, rightType); - SpatialRelatesFunction.SpatialRelations spatialRelations = spatialRelations(left, leftType, right, rightType); - try { - return spatialRelations.geometryRelatesGeometry(leftWKB, rightWKB); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static SpatialRelatesFunction.SpatialRelations spatialRelations( - Object left, - DataType leftType, - Object right, - DataType rightType - ) { - if (isSpatialGeo(leftType) || isSpatialGeo(rightType)) { - return getRelationsField("GEO"); - } else if (isSpatial(leftType) || isSpatial(rightType)) { - return getRelationsField("CARTESIAN"); - } else { - throw new IllegalArgumentException( - "Unsupported left and right types: left[" - + leftType.esType() - + ":" - + left.getClass().getSimpleName() - + "] right[" - + rightType.esType() - + ":" - + right.getClass().getSimpleName() - + "]" - ); - } - } - - private static BytesRef asGeometryWKB(Object object, DataType dataType) { - if (isString(dataType)) { - return SpatialCoordinateTypes.UNSPECIFIED.wktToWkb(object.toString()); - } else if (object instanceof BytesRef wkb) { - return wkb; - } else { - throw new IllegalArgumentException("Invalid geometry base type for " + dataType + ": " + object.getClass().getSimpleName()); - } - } - - private static boolean typeCompatible(DataType leftType, DataType rightType) { - if (isSpatial(leftType) && isSpatial(rightType)) { - // Both must be GEO_* or both must be CARTESIAN_* - return countGeo(leftType, rightType) != 1; - } - return true; - } - - private static DataType pickSpatialType(DataType leftType, DataType rightType) { - if (isSpatial(leftType)) { - return leftType; - } else if (isSpatial(rightType)) { - return rightType; - } else { - throw new IllegalArgumentException("Invalid spatial types: " + leftType + " and " + rightType); - } - } - - public static Matcher spatialEvaluatorString(DataType leftType, DataType rightType) { - String crsType = isSpatialGeo(pickSpatialType(leftType, rightType)) ? "Geo" : "Cartesian"; - return equalTo( - getFunctionClassName() + crsType + "SourceAndSourceEvaluator[leftValue=Attribute[channel=0], rightValue=Attribute[channel=1]]" - ); - } - - private static int countGeo(DataType... types) { - int count = 0; - for (DataType type : types) { - if (isSpatialGeo(type)) { - count++; - } - } - return count; + return typeErrorMessage(includeOrdinal, validPerPosition, types, false); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceTests.java new file mode 100644 index 0000000000000..c78977918fc5e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +@FunctionName("st_distance") +public class StDistanceTests extends BinarySpatialFunctionTestCase { + public StDistanceTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + DataType[] geoDataTypes = { DataType.GEO_POINT }; + StDistanceTests.addSpatialCombinations(suppliers, geoDataTypes); + DataType[] cartesianDataTypes = { DataType.CARTESIAN_POINT }; + StDistanceTests.addSpatialCombinations(suppliers, cartesianDataTypes); + return parameterSuppliersFromTypedData( + errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), StDistanceTests::typeErrorMessage) + ); + } + + @Override + protected Expression build(Source source, List args) { + return new StDistance(source, args.get(0), args.get(1)); + } + + protected static void addSpatialCombinations(List suppliers, DataType[] dataTypes) { + addSpatialCombinations(suppliers, dataTypes, DataType.DOUBLE, true); + } + + protected static String typeErrorMessage(boolean includeOrdinal, List> validPerPosition, List types) { + return typeErrorMessage(includeOrdinal, validPerPosition, types, true); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java index 5c794d707f5f4..f26e819685789 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java @@ -230,10 +230,16 @@ private void testCase( if (rewritesToMatchNone != YesNoSometimes.SOMETIMES) { assertThat(builder.stats().noNextScorer(), equalTo(0)); } + assertEqualsAndHashcodeStable(query, rewritten.toQuery(ctx)); } } } + private void assertEqualsAndHashcodeStable(Query query1, Query query2) { + assertEquals(query1, query2); + assertEquals(query1.hashCode(), query2.hashCode()); + } + private record StandardSetup(String fieldType, boolean multivaluedField, boolean empty, int count) implements Setup { @Override public XContentBuilder mapping(XContentBuilder builder) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index b3dbd97d495a9..14980df2f8789 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -39,6 +39,9 @@ import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsTaskSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.MultilingualE5SmallInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettings; @@ -109,6 +112,7 @@ public static List getNamedWriteables() { addAzureAiStudioNamedWriteables(namedWriteables); addGoogleAiStudioNamedWritables(namedWriteables); addMistralNamedWriteables(namedWriteables); + addCustomElandWriteables(namedWriteables); return namedWriteables; } @@ -349,4 +353,23 @@ private static void addInferenceResultsNamedWriteables(List namedWriteables) { + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + CustomElandInternalServiceSettings.NAME, + CustomElandInternalServiceSettings::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + CustomElandInternalTextEmbeddingServiceSettings.NAME, + CustomElandInternalTextEmbeddingServiceSettings::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(TaskSettings.class, CustomElandRerankTaskSettings.NAME, CustomElandRerankTaskSettings::new) + ); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index c2a4907125a31..dac0b2fae4b05 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -137,7 +137,7 @@ protected void merge(FieldMapper mergeWith, Conflicts conflicts, MapperMergeCont super.merge(mergeWith, conflicts, mapperMergeContext); conflicts.check(); var semanticMergeWith = (SemanticTextFieldMapper) mergeWith; - var context = mapperMergeContext.createChildContext(mergeWith.simpleName(), ObjectMapper.Dynamic.FALSE); + var context = mapperMergeContext.createChildContext(mergeWith.leafName(), ObjectMapper.Dynamic.FALSE); var inferenceField = inferenceFieldBuilder.apply(context.getMapperBuilderContext()); var mergedInferenceField = inferenceField.merge(semanticMergeWith.fieldType().getInferenceField(), context); inferenceFieldBuilder = c -> mergedInferenceField; @@ -146,16 +146,16 @@ protected void merge(FieldMapper mergeWith, Conflicts conflicts, MapperMergeCont @Override public SemanticTextFieldMapper build(MapperBuilderContext context) { if (copyTo.copyToFields().isEmpty() == false) { - throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name() + "] does not support [copy_to]"); + throw new IllegalArgumentException(CONTENT_TYPE + " field [" + leafName() + "] does not support [copy_to]"); } if (multiFieldsBuilder.hasMultiFields()) { - throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name() + "] does not support multi-fields"); + throw new IllegalArgumentException(CONTENT_TYPE + " field [" + leafName() + "] does not support multi-fields"); } - final String fullName = context.buildFullName(name()); - var childContext = context.createChildContext(name(), ObjectMapper.Dynamic.FALSE); + final String fullName = context.buildFullName(leafName()); + var childContext = context.createChildContext(leafName(), ObjectMapper.Dynamic.FALSE); final ObjectMapper inferenceField = inferenceFieldBuilder.apply(childContext); return new SemanticTextFieldMapper( - name(), + leafName(), new SemanticTextFieldType( fullName, inferenceId.getValue(), @@ -182,7 +182,7 @@ public Iterator iterator() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this); + return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this); } @Override @@ -221,7 +221,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (fieldType().getModelSettings() == null) { context.path().remove(); Builder builder = (Builder) new Builder( - simpleName(), + leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer() ).init(this); @@ -231,7 +231,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio .build(context.createDynamicMapperBuilderContext()); context.addDynamicMapper(mapper); } finally { - context.path().add(simpleName()); + context.path().add(leafName()); } } else { Conflicts conflicts = new Conflicts(fullFieldName); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java index 8d83a8a81ec0d..cb72b4d02302b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceFields.java @@ -17,6 +17,12 @@ public final class ServiceFields { public static final String MAX_INPUT_TOKENS = "max_input_tokens"; public static final String URL = "url"; public static final String MODEL_ID = "model_id"; + /** + * Represents the field elasticsearch uses to determine the embedding type (e.g. float, byte). + * The value this field is normally set to would be one of the values in + * {@link org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType} + */ + public static final String ELEMENT_TYPE = "element_type"; private ServiceFields() { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java new file mode 100644 index 0000000000000..bb4e0c2c513ac --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandEmbeddingModel.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; + +import java.util.Map; + +public class CustomElandEmbeddingModel extends CustomElandModel { + + public CustomElandEmbeddingModel( + String inferenceEntityId, + TaskType taskType, + String service, + Map serviceSettings, + ConfigurationParseContext context + ) { + this(inferenceEntityId, taskType, service, CustomElandInternalTextEmbeddingServiceSettings.fromMap(serviceSettings, context)); + } + + public CustomElandEmbeddingModel( + String inferenceEntityId, + TaskType taskType, + String service, + CustomElandInternalTextEmbeddingServiceSettings serviceSettings + ) { + super( + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings), + serviceSettings.getElasticsearchInternalServiceSettings() + ); + } + + @Override + public CustomElandInternalTextEmbeddingServiceSettings getServiceSettings() { + return (CustomElandInternalTextEmbeddingServiceSettings) super.getServiceSettings(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java index c62855c09cff2..cc52419f3f2d7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java @@ -38,7 +38,7 @@ public CustomElandInternalServiceSettings(int numAllocations, int numThreads, St * @param map Source map containing the config * @return The {@code CustomElandServiceSettings} builder */ - public static Builder fromMap(Map map) { + public static CustomElandInternalServiceSettings fromMap(Map map) { ValidationException validationException = new ValidationException(); Integer numAllocations = ServiceUtils.removeAsType(map, NUM_ALLOCATIONS, Integer.class); @@ -61,7 +61,7 @@ public CustomElandInternalServiceSettings build() { builder.setNumAllocations(numAllocations); builder.setNumThreads(numThreads); builder.setModelId(modelId); - return builder; + return builder.build(); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java new file mode 100644 index 0000000000000..5ef9ce1a0507f --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettings.java @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.ELEMENT_TYPE; +import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; + +public class CustomElandInternalTextEmbeddingServiceSettings implements ServiceSettings { + + public static final String NAME = "custom_eland_model_internal_text_embedding_service_settings"; + + /** + * Parse the CustomElandServiceSettings from map and validate the setting values. + * + * This method does not verify the model variant + * + * If required setting are missing or the values are invalid an + * {@link ValidationException} is thrown. + * + * @param map Source map containing the config + * @param context The parser context, whether it is from an HTTP request or from persistent storage + * @return The {@code CustomElandServiceSettings} builder + */ + public static CustomElandInternalTextEmbeddingServiceSettings fromMap(Map map, ConfigurationParseContext context) { + return switch (context) { + case REQUEST -> fromRequestMap(map); + case PERSISTENT -> fromPersistedMap(map); + }; + } + + private static CustomElandInternalTextEmbeddingServiceSettings fromRequestMap(Map map) { + ValidationException validationException = new ValidationException(); + var commonFields = commonFieldsFromMap(map, validationException); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new CustomElandInternalTextEmbeddingServiceSettings(commonFields); + } + + private static CustomElandInternalTextEmbeddingServiceSettings fromPersistedMap(Map map) { + var commonFields = commonFieldsFromMap(map); + Integer dims = extractOptionalPositiveInteger(map, DIMENSIONS, ModelConfigurations.SERVICE_SETTINGS, new ValidationException()); + + return new CustomElandInternalTextEmbeddingServiceSettings(commonFields, dims); + } + + private record CommonFields( + ElasticsearchInternalServiceSettings internalServiceSettings, + SimilarityMeasure similarityMeasure, + DenseVectorFieldMapper.ElementType elementType + ) {} + + private static CommonFields commonFieldsFromMap(Map map) { + return commonFieldsFromMap(map, new ValidationException()); + } + + private static CommonFields commonFieldsFromMap(Map map, ValidationException validationException) { + var internalSettings = ElasticsearchInternalServiceSettings.fromMap(map, validationException); + SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException); + DenseVectorFieldMapper.ElementType elementType = extractOptionalEnum( + map, + ELEMENT_TYPE, + ModelConfigurations.SERVICE_SETTINGS, + DenseVectorFieldMapper.ElementType::fromString, + EnumSet.of(DenseVectorFieldMapper.ElementType.BYTE, DenseVectorFieldMapper.ElementType.FLOAT), + validationException + ); + + return new CommonFields( + internalSettings, + Objects.requireNonNullElse(similarity, SimilarityMeasure.COSINE), + Objects.requireNonNullElse(elementType, DenseVectorFieldMapper.ElementType.FLOAT) + ); + } + + private final ElasticsearchInternalServiceSettings internalServiceSettings; + private final Integer dimensions; + private final SimilarityMeasure similarityMeasure; + private final DenseVectorFieldMapper.ElementType elementType; + + public CustomElandInternalTextEmbeddingServiceSettings(int numAllocations, int numThreads, String modelId) { + this(numAllocations, numThreads, modelId, null, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); + } + + public CustomElandInternalTextEmbeddingServiceSettings( + int numAllocations, + int numThreads, + String modelId, + Integer dimensions, + SimilarityMeasure similarityMeasure, + DenseVectorFieldMapper.ElementType elementType + ) { + internalServiceSettings = new ElasticsearchInternalServiceSettings(numAllocations, numThreads, modelId); + this.dimensions = dimensions; + this.similarityMeasure = Objects.requireNonNull(similarityMeasure); + this.elementType = Objects.requireNonNull(elementType); + } + + public CustomElandInternalTextEmbeddingServiceSettings(StreamInput in) throws IOException { + internalServiceSettings = new ElasticsearchInternalServiceSettings(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ELAND_SETTINGS_ADDED)) { + dimensions = in.readOptionalVInt(); + similarityMeasure = in.readEnum(SimilarityMeasure.class); + elementType = in.readEnum(DenseVectorFieldMapper.ElementType.class); + } else { + dimensions = null; + similarityMeasure = SimilarityMeasure.COSINE; + elementType = DenseVectorFieldMapper.ElementType.FLOAT; + } + } + + private CustomElandInternalTextEmbeddingServiceSettings(CommonFields commonFields) { + this(commonFields, null); + } + + private CustomElandInternalTextEmbeddingServiceSettings(CommonFields commonFields, Integer dimensions) { + internalServiceSettings = commonFields.internalServiceSettings; + this.dimensions = dimensions; + similarityMeasure = commonFields.similarityMeasure; + elementType = commonFields.elementType; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + internalServiceSettings.addXContentFragment(builder, params); + + if (dimensions != null) { + builder.field(DIMENSIONS, dimensions); + } + + if (similarityMeasure != null) { + builder.field(SIMILARITY, similarityMeasure); + } + + if (elementType != null) { + builder.field(ELEMENT_TYPE, elementType); + } + + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return CustomElandInternalTextEmbeddingServiceSettings.NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.V_8_13_0; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + internalServiceSettings.writeTo(out); + + if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ELAND_SETTINGS_ADDED)) { + out.writeOptionalVInt(dimensions); + out.writeEnum(similarityMeasure); + out.writeEnum(elementType); + } + } + + public ElasticsearchInternalServiceSettings getElasticsearchInternalServiceSettings() { + return internalServiceSettings; + } + + @Override + public DenseVectorFieldMapper.ElementType elementType() { + return elementType; + } + + @Override + public SimilarityMeasure similarity() { + return similarityMeasure; + } + + @Override + public Integer dimensions() { + return dimensions; + } + + @Override + public ToXContentObject getFilteredXContentObject() { + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CustomElandInternalTextEmbeddingServiceSettings that = (CustomElandInternalTextEmbeddingServiceSettings) o; + return Objects.equals(internalServiceSettings, that.internalServiceSettings) + && Objects.equals(dimensions, that.dimensions) + && Objects.equals(similarityMeasure, that.similarityMeasure) + && Objects.equals(elementType, that.elementType); + } + + @Override + public int hashCode() { + return Objects.hash(internalServiceSettings, dimensions, similarityMeasure, elementType); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java index 1f9ec163aa546..5a82e73299b85 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandModel.java @@ -9,71 +9,41 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.TaskSettings; -import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; -import java.util.Map; +import java.util.Objects; import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED; -public class CustomElandModel extends ElasticsearchModel { +public class CustomElandModel extends Model implements ElasticsearchModel { + private final InternalServiceSettings internalServiceSettings; - public static CustomElandModel build( - String inferenceEntityId, - TaskType taskType, - String service, - CustomElandInternalServiceSettings serviceSettings, - @Nullable TaskSettings taskSettings - ) { - return taskSettings == null - ? new CustomElandModel(inferenceEntityId, taskType, service, serviceSettings) - : new CustomElandModel(inferenceEntityId, taskType, service, serviceSettings, taskSettings); - } - - public CustomElandModel( - String inferenceEntityId, - TaskType taskType, - String service, - CustomElandInternalServiceSettings serviceSettings - ) { - super(inferenceEntityId, taskType, service, serviceSettings); + public CustomElandModel(ModelConfigurations configurations, InternalServiceSettings internalServiceSettings) { + super(configurations); + this.internalServiceSettings = Objects.requireNonNull(internalServiceSettings); } - private CustomElandModel( - String inferenceEntityId, - TaskType taskType, - String service, - CustomElandInternalServiceSettings serviceSettings, - TaskSettings taskSettings - ) { - super(inferenceEntityId, taskType, service, serviceSettings, taskSettings); + public String getModelId() { + return internalServiceSettings.getModelId(); } @Override - public CustomElandInternalServiceSettings getServiceSettings() { - return (CustomElandInternalServiceSettings) super.getServiceSettings(); - } - - @Override - StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { - var startRequest = new StartTrainedModelDeploymentAction.Request( - this.getServiceSettings().getModelId(), - this.getInferenceEntityId() - ); - startRequest.setNumberOfAllocations(this.getServiceSettings().getNumAllocations()); - startRequest.setThreadsPerAllocation(this.getServiceSettings().getNumThreads()); + public StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { + var startRequest = new StartTrainedModelDeploymentAction.Request(internalServiceSettings.getModelId(), this.getInferenceEntityId()); + startRequest.setNumberOfAllocations(internalServiceSettings.getNumAllocations()); + startRequest.setThreadsPerAllocation(internalServiceSettings.getNumThreads()); startRequest.setWaitForState(STARTED); return startRequest; } @Override - ActionListener getCreateTrainedModelAssignmentActionListener( + public ActionListener getCreateTrainedModelAssignmentActionListener( Model model, ActionListener listener ) { @@ -92,7 +62,7 @@ public void onFailure(Exception e) { "Could not start the TextEmbeddingService service as the " + "custom eland model [{0}] for this platform cannot be found." + " Custom models need to be loaded into the cluster with eland before they can be started.", - getServiceSettings().getModelId() + getModelId() ) ); return; @@ -101,12 +71,4 @@ public void onFailure(Exception e) { } }; } - - public static TaskSettings taskSettingsFromMap(TaskType taskType, Map taskSettingsMap) { - if (TaskType.RERANK.equals(taskType)) { - return CustomElandRerankTaskSettings.defaultsFromMap(taskSettingsMap); - } - - return null; - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java new file mode 100644 index 0000000000000..d880450739319 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; + +import java.util.Map; + +public class CustomElandRerankModel extends CustomElandModel { + + public CustomElandRerankModel( + String inferenceEntityId, + TaskType taskType, + String service, + Map serviceSettings, + Map taskSettings, + ConfigurationParseContext context + ) { + this( + inferenceEntityId, + taskType, + service, + CustomElandInternalServiceSettings.fromMap(serviceSettings), + CustomElandRerankTaskSettings.defaultsFromMap(taskSettings) + ); + } + + // default for testing + CustomElandRerankModel( + String inferenceEntityId, + TaskType taskType, + String service, + CustomElandInternalServiceSettings serviceSettings, + CustomElandRerankTaskSettings taskSettings + ) { + super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), serviceSettings); + } + + @Override + public CustomElandInternalServiceSettings getServiceSettings() { + return (CustomElandInternalServiceSettings) super.getServiceSettings(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index dbc36960a8231..d5401f61823db 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -46,6 +46,8 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; import java.io.IOException; @@ -127,15 +129,17 @@ private void customElandCase( + "]. You may need to load it into the cluster using eland." ); } else { - var customElandInternalServiceSettings = (CustomElandInternalServiceSettings) CustomElandInternalServiceSettings.fromMap( - serviceSettingsMap - ).build(); - throwIfNotEmptyMap(serviceSettingsMap, name()); + var model = createCustomElandModel( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + ConfigurationParseContext.REQUEST + ); - var taskSettings = CustomElandModel.taskSettingsFromMap(taskType, taskSettingsMap); + throwIfNotEmptyMap(serviceSettingsMap, name()); throwIfNotEmptyMap(taskSettingsMap, name()); - var model = CustomElandModel.build(inferenceEntityId, taskType, name(), customElandInternalServiceSettings, taskSettings); delegate.onResponse(model); } }); @@ -143,6 +147,20 @@ private void customElandCase( client.execute(GetTrainedModelsAction.INSTANCE, request, getModelsListener); } + private static CustomElandModel createCustomElandModel( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + ConfigurationParseContext context + ) { + return switch (taskType) { + case TEXT_EMBEDDING -> new CustomElandEmbeddingModel(inferenceEntityId, taskType, NAME, serviceSettings, context); + case RERANK -> new CustomElandRerankModel(inferenceEntityId, taskType, NAME, serviceSettings, taskSettings, context); + default -> throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); + }; + } + private void e5Case( String inferenceEntityId, TaskType taskType, @@ -187,7 +205,7 @@ private static boolean modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgno } @Override - public ElasticsearchModel parsePersistedConfigWithSecrets( + public Model parsePersistedConfigWithSecrets( String inferenceEntityId, TaskType taskType, Map config, @@ -197,7 +215,7 @@ public ElasticsearchModel parsePersistedConfigWithSecrets( } @Override - public ElasticsearchModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { + public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMap(config, ModelConfigurations.TASK_SETTINGS); @@ -214,14 +232,58 @@ public ElasticsearchModel parsePersistedConfig(String inferenceEntityId, TaskTyp (MultilingualE5SmallInternalServiceSettings) MultilingualE5SmallInternalServiceSettings.fromMap(serviceSettingsMap).build() ); } else { - var serviceSettings = (CustomElandInternalServiceSettings) CustomElandInternalServiceSettings.fromMap(serviceSettingsMap) - .build(); - var taskSettings = CustomElandModel.taskSettingsFromMap(taskType, taskSettingsMap); + return createCustomElandModel( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + ConfigurationParseContext.PERSISTENT + ); + } + } + + @Override + public void checkModelConfig(Model model, ActionListener listener) { + if (model instanceof CustomElandEmbeddingModel elandModel && elandModel.getTaskType() == TaskType.TEXT_EMBEDDING) { + // At this point the inference endpoint configuration has not been persisted yet, if we attempt to do inference using the + // inference id we'll get an error because the trained model code needs to use the persisted inference endpoint to retrieve the + // model id. To get around this we'll have the getEmbeddingSize() method use the model id instead of inference id. So we need + // to create a temporary model that overrides the inference id with the model id. + var temporaryModelWithModelId = new CustomElandEmbeddingModel( + elandModel.getModelId(), + elandModel.getTaskType(), + elandModel.getConfigurations().getService(), + elandModel.getServiceSettings() + ); - return CustomElandModel.build(inferenceEntityId, taskType, name(), serviceSettings, taskSettings); + ServiceUtils.getEmbeddingSize( + temporaryModelWithModelId, + this, + listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(elandModel, size))) + ); + } else { + listener.onResponse(model); } } + private static CustomElandEmbeddingModel updateModelWithEmbeddingDetails(CustomElandEmbeddingModel model, int embeddingSize) { + CustomElandInternalTextEmbeddingServiceSettings serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings( + model.getServiceSettings().getElasticsearchInternalServiceSettings().getNumAllocations(), + model.getServiceSettings().getElasticsearchInternalServiceSettings().getNumThreads(), + model.getServiceSettings().getElasticsearchInternalServiceSettings().getModelId(), + embeddingSize, + model.getServiceSettings().similarity(), + model.getServiceSettings().elementType() + ); + + return new CustomElandEmbeddingModel( + model.getInferenceEntityId(), + model.getTaskType(), + model.getConfigurations().getService(), + serviceSettings + ); + } + @Override public void infer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index a384dfe9a2c90..c2cf17fcc19f1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -9,14 +9,43 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings; import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; public class ElasticsearchInternalServiceSettings extends InternalServiceSettings { public static final String NAME = "text_embedding_internal_service_settings"; + private static final int FAILED_INT_PARSE_VALUE = -1; + + public static ElasticsearchInternalServiceSettings fromMap(Map map, ValidationException validationException) { + Integer numAllocations = extractOptionalPositiveInteger( + map, + NUM_ALLOCATIONS, + ModelConfigurations.SERVICE_SETTINGS, + validationException + ); + Integer numThreads = extractOptionalPositiveInteger(map, NUM_THREADS, ModelConfigurations.SERVICE_SETTINGS, validationException); + validateParameters(numAllocations, validationException, numThreads); + + String modelId = ServiceUtils.extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + + // if an error occurred while parsing, we'll set these to an invalid value so we don't accidentally get a + // null pointer when doing unboxing + return new ElasticsearchInternalServiceSettings( + Objects.requireNonNullElse(numAllocations, FAILED_INT_PARSE_VALUE), + Objects.requireNonNullElse(numThreads, FAILED_INT_PARSE_VALUE), + modelId + ); + } public ElasticsearchInternalServiceSettings(int numAllocations, int numThreads, String modelVariant) { super(numAllocations, numThreads, modelVariant); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java index dc6561ba992fe..627e570b24163 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchModel.java @@ -9,41 +9,15 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; -import org.elasticsearch.inference.TaskSettings; -import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; -public abstract class ElasticsearchModel extends Model { +public interface ElasticsearchModel { + String getModelId(); - public ElasticsearchModel( - String inferenceEntityId, - TaskType taskType, - String service, - ElasticsearchInternalServiceSettings serviceSettings - ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings)); - } + StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest(); - public ElasticsearchModel( - String inferenceEntityId, - TaskType taskType, - String service, - ElasticsearchInternalServiceSettings serviceSettings, - TaskSettings taskSettings - ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings)); - } - - @Override - public ElasticsearchInternalServiceSettings getServiceSettings() { - return (ElasticsearchInternalServiceSettings) super.getServiceSettings(); - } - - abstract StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest(); - - abstract ActionListener getCreateTrainedModelAssignmentActionListener( + ActionListener getCreateTrainedModelAssignmentActionListener( Model model, ActionListener listener ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java index 86be6a04b213d..60d68eb2fcee7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/MultilingualE5SmallModel.java @@ -10,6 +10,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; @@ -17,7 +18,7 @@ import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED; -public class MultilingualE5SmallModel extends ElasticsearchModel { +public class MultilingualE5SmallModel extends Model implements ElasticsearchModel { public MultilingualE5SmallModel( String inferenceEntityId, @@ -25,7 +26,7 @@ public MultilingualE5SmallModel( String service, MultilingualE5SmallInternalServiceSettings serviceSettings ) { - super(inferenceEntityId, taskType, service, serviceSettings); + super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings)); } @Override @@ -34,7 +35,12 @@ public MultilingualE5SmallInternalServiceSettings getServiceSettings() { } @Override - StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { + public String getModelId() { + return getServiceSettings().getModelId(); + } + + @Override + public StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRequest() { var startRequest = new StartTrainedModelDeploymentAction.Request( this.getServiceSettings().getModelId(), this.getInferenceEntityId() @@ -47,7 +53,7 @@ StartTrainedModelDeploymentAction.Request getStartTrainedModelDeploymentActionRe } @Override - ActionListener getCreateTrainedModelAssignmentActionListener( + public ActionListener getCreateTrainedModelAssignmentActionListener( Model model, ActionListener listener ) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index 161ab6c47bfeb..6e311c39c787a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker; @@ -78,9 +79,14 @@ public void checkModelConfig(Model model, ActionListener listener) { } private static HuggingFaceEmbeddingsModel updateModelWithEmbeddingDetails(HuggingFaceEmbeddingsModel model, int embeddingSize) { + // default to cosine similarity + var similarity = model.getServiceSettings().similarity() == null + ? SimilarityMeasure.COSINE + : model.getServiceSettings().similarity(); + var serviceSettings = new HuggingFaceServiceSettings( model.getServiceSettings().uri(), - model.getServiceSettings().similarity(), // we don't know the similarity but use whatever the user specified + similarity, embeddingSize, model.getTokenLimit(), model.getServiceSettings().rateLimitSettings() diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java index ee7db662b4997..ad9ccd17a95c4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/InternalServiceSettings.java @@ -81,11 +81,15 @@ public int hashCode() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + addXContentFragment(builder, params); + builder.endObject(); + return builder; + } + + public void addXContentFragment(XContentBuilder builder, Params params) throws IOException { builder.field(NUM_ALLOCATIONS, getNumAllocations()); builder.field(NUM_THREADS, getNumThreads()); builder.field(MODEL_ID, getModelId()); - builder.endObject(); - return builder; } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java new file mode 100644 index 0000000000000..0cc3e6698388d --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalTextEmbeddingServiceSettingsTests.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceFields; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.ELEMENT_TYPE; +import static org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings.NUM_ALLOCATIONS; +import static org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings.NUM_THREADS; +import static org.hamcrest.Matchers.is; + +public class CustomElandInternalTextEmbeddingServiceSettingsTests extends AbstractWireSerializingTestCase< + CustomElandInternalTextEmbeddingServiceSettings> { + + public static CustomElandInternalTextEmbeddingServiceSettings createRandom() { + var numAllocations = randomIntBetween(1, 10); + var numThreads = randomIntBetween(1, 10); + var modelId = randomAlphaOfLength(8); + SimilarityMeasure similarityMeasure = SimilarityMeasure.COSINE; + Integer dims = null; + var setDimensions = randomBoolean(); + if (setDimensions) { + dims = 123; + } + + var elementType = randomFrom(DenseVectorFieldMapper.ElementType.values()); + + return new CustomElandInternalTextEmbeddingServiceSettings( + numAllocations, + numThreads, + modelId, + dims, + similarityMeasure, + elementType + ); + } + + public void testFromMap_Request_CreatesSettingsCorrectly() { + var modelId = "model-foo"; + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var numAllocations = 1; + var numThreads = 1; + var serviceSettings = CustomElandInternalTextEmbeddingServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.MODEL_ID, + modelId, + NUM_ALLOCATIONS, + numAllocations, + NUM_THREADS, + numThreads, + ServiceFields.SIMILARITY, + similarity, + ELEMENT_TYPE, + DenseVectorFieldMapper.ElementType.FLOAT.toString() + ) + ), + ConfigurationParseContext.REQUEST + ); + + assertThat( + serviceSettings, + is( + new CustomElandInternalTextEmbeddingServiceSettings( + numAllocations, + numThreads, + modelId, + null, + SimilarityMeasure.DOT_PRODUCT, + DenseVectorFieldMapper.ElementType.FLOAT + ) + ) + ); + } + + public void testFromMap_Request_DoesNotDefaultSimilarityElementType() { + var modelId = "model-foo"; + var numAllocations = 1; + var numThreads = 1; + var serviceSettings = CustomElandInternalTextEmbeddingServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId, NUM_ALLOCATIONS, numAllocations, NUM_THREADS, numThreads)), + ConfigurationParseContext.REQUEST + ); + + assertThat( + serviceSettings, + is( + new CustomElandInternalTextEmbeddingServiceSettings( + numAllocations, + numThreads, + modelId, + null, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ) + ) + ); + } + + public void testFromMap_Request_IgnoresDimensions() { + var modelId = "model-foo"; + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var numAllocations = 1; + var numThreads = 1; + var serviceSettings = CustomElandInternalTextEmbeddingServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.MODEL_ID, + modelId, + NUM_ALLOCATIONS, + numAllocations, + NUM_THREADS, + numThreads, + ServiceFields.SIMILARITY, + similarity, + ELEMENT_TYPE, + DenseVectorFieldMapper.ElementType.FLOAT.toString(), + ServiceFields.DIMENSIONS, + 1 + ) + ), + ConfigurationParseContext.REQUEST + ); + + assertThat( + serviceSettings, + is( + new CustomElandInternalTextEmbeddingServiceSettings( + numAllocations, + numThreads, + modelId, + null, + SimilarityMeasure.DOT_PRODUCT, + DenseVectorFieldMapper.ElementType.FLOAT + ) + ) + ); + } + + public void testFromMap_Persistent_CreatesSettingsCorrectly() { + var modelId = "model-foo"; + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var numAllocations = 1; + var numThreads = 1; + var serviceSettings = CustomElandInternalTextEmbeddingServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.MODEL_ID, + modelId, + NUM_ALLOCATIONS, + numAllocations, + NUM_THREADS, + numThreads, + ServiceFields.SIMILARITY, + similarity, + ELEMENT_TYPE, + DenseVectorFieldMapper.ElementType.FLOAT.toString(), + ServiceFields.DIMENSIONS, + 1 + ) + ), + ConfigurationParseContext.PERSISTENT + ); + + assertThat( + serviceSettings, + is( + new CustomElandInternalTextEmbeddingServiceSettings( + numAllocations, + numThreads, + modelId, + 1, + SimilarityMeasure.DOT_PRODUCT, + DenseVectorFieldMapper.ElementType.FLOAT + ) + ) + ); + } + + public void testToXContent_WritesAllValues() throws IOException { + var entity = new CustomElandInternalTextEmbeddingServiceSettings( + 1, + 1, + "model_id", + 100, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.BYTE + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, is(""" + {"num_allocations":1,"num_threads":1,"model_id":"model_id","dimensions":100,"similarity":"cosine","element_type":"byte"}""")); + } + + @Override + protected Writeable.Reader instanceReader() { + return CustomElandInternalTextEmbeddingServiceSettings::new; + } + + @Override + protected CustomElandInternalTextEmbeddingServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected CustomElandInternalTextEmbeddingServiceSettings mutateInstance(CustomElandInternalTextEmbeddingServiceSettings instance) + throws IOException { + return randomValueOtherThan(instance, CustomElandInternalTextEmbeddingServiceSettingsTests::createRandom); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index dfcfe466c2a3b..3bec202ed9e5e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceResults; @@ -40,6 +41,7 @@ import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResultsTests; +import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.core.utils.FloatConversionUtils; @@ -52,6 +54,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -66,6 +69,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -286,9 +290,9 @@ public void testParseRequestConfig_Rerank() { ); ActionListener modelListener = ActionListener.wrap(model -> { - assertThat(model, instanceOf(CustomElandModel.class)); + assertThat(model, instanceOf(CustomElandRerankModel.class)); assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); - assertThat(model.getServiceSettings(), instanceOf(ElasticsearchInternalServiceSettings.class)); + assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class)); assertEquals(returnDocs, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); }, e -> { fail("Model parsing failed " + e.getMessage()); }); @@ -328,9 +332,9 @@ public void testParseRequestConfig_Rerank_DefaultTaskSettings() { ); ActionListener modelListener = ActionListener.wrap(model -> { - assertThat(model, instanceOf(CustomElandModel.class)); + assertThat(model, instanceOf(CustomElandRerankModel.class)); assertThat(model.getTaskSettings(), instanceOf(CustomElandRerankTaskSettings.class)); - assertThat(model.getServiceSettings(), instanceOf(ElasticsearchInternalServiceSettings.class)); + assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class)); assertEquals(Boolean.TRUE, ((CustomElandRerankTaskSettings) model.getTaskSettings()).returnDocuments()); }, e -> { fail("Model parsing failed " + e.getMessage()); }); @@ -391,10 +395,14 @@ public void testParsePersistedConfig() { ) ); - CustomElandModel parsedModel = (CustomElandModel) service.parsePersistedConfig(randomInferenceEntityId, taskType, settings); - var elandServiceSettings = new CustomElandInternalServiceSettings(1, 4, "invalid"); + CustomElandEmbeddingModel parsedModel = (CustomElandEmbeddingModel) service.parsePersistedConfig( + randomInferenceEntityId, + taskType, + settings + ); + var elandServiceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "invalid"); assertEquals( - new CustomElandModel(randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, elandServiceSettings), + new CustomElandEmbeddingModel(randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, elandServiceSettings), parsedModel ); } @@ -683,7 +691,7 @@ public void testParseRequestConfigEland_PreservesTaskType() { new GetTrainedModelsAction.Response(new QueryPage<>(List.of(mock(TrainedModelConfig.class)), 1, mock(ParseField.class))) ); return Void.TYPE; - }).when(client).execute(any(), any(), any()); + }).when(client).execute(eq(GetTrainedModelsAction.INSTANCE), any(), any()); when(client.threadPool()).thenReturn(threadPool); var service = createService(client); @@ -702,16 +710,8 @@ public void testParseRequestConfigEland_PreservesTaskType() { ) ); - var serviceSettings = new CustomElandInternalServiceSettings(1, 4, "custom-model"); - var taskType = randomFrom(TaskType.values()); - var taskSettings = taskType == TaskType.RERANK ? CustomElandRerankTaskSettings.DEFAULT_SETTINGS : null; - var expectedModel = CustomElandModel.build( - randomInferenceEntityId, - taskType, - ElasticsearchInternalService.NAME, - serviceSettings, - taskSettings - ); + var taskType = randomFrom(EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING)); + CustomElandModel expectedModel = getCustomElandModel(taskType); PlainActionFuture listener = new PlainActionFuture<>(); service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), listener); @@ -719,6 +719,29 @@ public void testParseRequestConfigEland_PreservesTaskType() { assertThat(model, is(expectedModel)); } + private CustomElandModel getCustomElandModel(TaskType taskType) { + CustomElandModel expectedModel = null; + if (taskType == TaskType.RERANK) { + expectedModel = new CustomElandRerankModel( + randomInferenceEntityId, + taskType, + ElasticsearchInternalService.NAME, + new CustomElandInternalServiceSettings(1, 4, "custom-model"), + CustomElandRerankTaskSettings.DEFAULT_SETTINGS + ); + } else if (taskType == TaskType.TEXT_EMBEDDING) { + var serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "custom-model"); + + expectedModel = new CustomElandEmbeddingModel( + randomInferenceEntityId, + taskType, + ElasticsearchInternalService.NAME, + serviceSettings + ); + } + return expectedModel; + } + public void testBuildInferenceRequest() { var id = randomAlphaOfLength(5); var inputs = randomList(1, 3, () -> randomAlphaOfLength(4)); @@ -782,6 +805,61 @@ public void onFailure(Exception e) { assertEquals("text_field", putConfig.getInput().getFieldNames().get(0)); } + public void testParseRequestConfigEland_SetsDimensionsToOne() { + var client = mock(Client.class); + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocationOnMock + .getArguments()[2]; + listener.onResponse( + new InferModelAction.Response(List.of(new MlTextEmbeddingResults("field", new double[] { 0.1 }, false)), "id", true) + ); + + var request = (InferModelAction.Request) invocationOnMock.getArguments()[1]; + assertThat(request.getId(), is("custom-model")); + return Void.TYPE; + }).when(client).execute(eq(InferModelAction.INSTANCE), any(), any()); + when(client.threadPool()).thenReturn(threadPool); + + var service = createService(client); + + var serviceSettings = new CustomElandInternalTextEmbeddingServiceSettings( + 1, + 4, + "custom-model", + 1, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); + var taskType = TaskType.TEXT_EMBEDDING; + var expectedModel = new CustomElandEmbeddingModel( + randomInferenceEntityId, + taskType, + ElasticsearchInternalService.NAME, + serviceSettings + ); + + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig( + new CustomElandEmbeddingModel( + randomInferenceEntityId, + taskType, + ElasticsearchInternalService.NAME, + new CustomElandInternalTextEmbeddingServiceSettings( + 1, + 4, + "custom-model", + null, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ) + ), + listener + ); + var model = listener.actionGet(TimeValue.THIRTY_SECONDS); + assertThat(model, is(expectedModel)); + } + private ElasticsearchInternalService createService(Client client) { var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client); return new ElasticsearchInternalService(context); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index de5c7ec83d57e..14fe1451ebace 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -529,12 +529,15 @@ public void testCheckModelConfig_IncludesMaxTokens() throws IOException { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var model = HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret", 1); + var model = HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret", 1, 1, SimilarityMeasure.DOT_PRODUCT); PlainActionFuture listener = new PlainActionFuture<>(); service.checkModelConfig(model, listener); var result = listener.actionGet(TIMEOUT); - assertThat(result, is(HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret", 1, 1))); + assertThat( + result, + is(HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret", 1, 1, SimilarityMeasure.DOT_PRODUCT)) + ); } } @@ -566,7 +569,7 @@ public void testCheckModelConfig_UsesUserSpecifiedSimilarity() throws IOExceptio } } - public void testCheckModelConfig_LeavesSimilarityAsNull_WhenUnspecified() throws IOException { + public void testCheckModelConfig_DefaultsSimilarityToCosine() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new HuggingFaceService(senderFactory, createWithEmptySettings(threadPool))) { @@ -587,11 +590,13 @@ public void testCheckModelConfig_LeavesSimilarityAsNull_WhenUnspecified() throws service.checkModelConfig(model, listener); var result = listener.actionGet(TIMEOUT); - assertThat(result, is(HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret", 1, 1, null))); + assertThat( + result, + is(HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret", 1, 1, SimilarityMeasure.COSINE)) + ); } } - // TODO public void testChunkedInfer_CallsInfer_TextEmbedding_ConvertsFloatResponse() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 33efabf101be7..92c45c51f0516 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -143,7 +143,7 @@ public static final class Builder extends FieldMapper.Builder { return parsedMetrics; }, m -> toType(m).metrics, XContentBuilder::enumSet, Objects::toString).addValidator(v -> { if (v == null || v.isEmpty()) { - throw new IllegalArgumentException("Property [" + Names.METRICS + "] is required for field [" + name() + "]."); + throw new IllegalArgumentException("Property [" + Names.METRICS + "] is required for field [" + leafName() + "]."); } }); @@ -209,21 +209,23 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) { } if (metrics.getValue().contains(defaultMetric.getValue()) == false) { - throw new IllegalArgumentException("Property [" + Names.DEFAULT_METRIC + "] is required for field [" + name() + "]."); + throw new IllegalArgumentException( + "Property [" + Names.DEFAULT_METRIC + "] is required for field [" + leafName() + "]." + ); } } if (metrics.getValue().contains(defaultMetric.getValue()) == false) { // The default_metric is not defined in the "metrics" field throw new IllegalArgumentException( - "Default metric [" + defaultMetric.getValue() + "] is not defined in the metrics of field [" + name() + "]." + "Default metric [" + defaultMetric.getValue() + "] is not defined in the metrics of field [" + leafName() + "]." ); } EnumMap metricMappers = new EnumMap<>(Metric.class); // Instantiate one NumberFieldMapper instance for each metric for (Metric m : this.metrics.getValue()) { - String fieldName = subfieldName(name(), m); + String fieldName = subfieldName(leafName(), m); NumberFieldMapper.Builder builder; if (m == Metric.value_count) { @@ -259,14 +261,14 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) { }, () -> new EnumMap<>(Metric.class))); AggregateDoubleMetricFieldType metricFieldType = new AggregateDoubleMetricFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), meta.getValue(), timeSeriesMetric.getValue() ); metricFieldType.setMetricFields(metricFields); metricFieldType.setDefaultMetric(defaultMetric.getValue()); - return new AggregateDoubleMetricFieldMapper(name(), metricFieldType, metricMappers, this); + return new AggregateDoubleMetricFieldMapper(leafName(), metricFieldType, metricMappers, this); } } @@ -586,7 +588,7 @@ protected boolean supportsParsingObject() { @Override protected void parseCreateField(DocumentParserContext context) throws IOException { - context.path().add(simpleName()); + context.path().add(leafName()); XContentParser.Token token; XContentSubParser subParser = null; EnumMap metricsParsed = new EnumMap<>(Metric.class); @@ -705,7 +707,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode).metric(metricType).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode).metric(metricType).init(this); } @Override @@ -716,9 +718,9 @@ protected SyntheticSourceMode syntheticSourceMode() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { return new CompositeSyntheticFieldLoader( - simpleName(), + leafName(), name(), - new AggregateMetricSyntheticFieldLoader(name(), simpleName(), metrics), + new AggregateMetricSyntheticFieldLoader(name(), leafName(), metrics), new CompositeSyntheticFieldLoader.MalformedValuesLayer(name()) ); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index c27b3c8207102..dedbfe8275f19 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -74,7 +74,7 @@ private static ConstantKeywordFieldMapper toType(FieldMapper in) { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } public static class Builder extends FieldMapper.Builder { @@ -110,8 +110,8 @@ public ConstantKeywordFieldMapper build(MapperBuilderContext context) { ); } return new ConstantKeywordFieldMapper( - name(), - new ConstantKeywordFieldType(context.buildFullName(name()), value.getValue(), meta.getValue()) + leafName(), + new ConstantKeywordFieldType(context.buildFullName(leafName()), value.getValue(), meta.getValue()) ); } } @@ -321,7 +321,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (fieldType().value == null) { ConstantKeywordFieldType newFieldType = new ConstantKeywordFieldType(fieldType().name(), value, fieldType().meta()); - Mapper update = new ConstantKeywordFieldMapper(simpleName(), newFieldType); + Mapper update = new ConstantKeywordFieldMapper(leafName(), newFieldType); boolean dynamicMapperAdded = context.addDynamicMapper(update); // the mapper is already part of the mapping, we're just updating it with the new value assert dynamicMapperAdded; @@ -374,7 +374,7 @@ public boolean hasValue() { @Override public void write(XContentBuilder b) throws IOException { if (fieldType().value != null) { - b.field(simpleName(), fieldType().value); + b.field(leafName(), fieldType().value); } } diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java index 0025d3f7dd7b2..16c2eb3e6c51c 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java @@ -290,16 +290,16 @@ protected Parameter[] getParameters() { public FieldMapper build(MapperBuilderContext context) { BinaryFieldMapper countFieldMapper = new BinaryFieldMapper.Builder( - name() + COUNT_FIELD_NAME_SUFFIX, + leafName() + COUNT_FIELD_NAME_SUFFIX, context.isSourceSynthetic() ).docValues(true).build(context); boolean isIndexed = indexed.getValue(); FieldType ft = isIndexed ? FIELD_TYPE_INDEXED : FIELD_TYPE_NOT_INDEXED; return new CountedKeywordFieldMapper( - name(), + leafName(), ft, new CountedKeywordFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), isIndexed, false, true, @@ -401,7 +401,7 @@ public Iterator iterator() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index e356fc2756c56..9da04ef3db114 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -158,7 +158,7 @@ private String parseNullValueAsString(Object o) { parseUnsignedLong(o); // confirm that null_value is a proper unsigned_long return (o instanceof BytesRef) ? ((BytesRef) o).utf8ToString() : o.toString(); } catch (Exception e) { - throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e); + throw new MapperParsingException("Error parsing [null_value] on field [" + leafName() + "]: " + e.getMessage(), e); } } @@ -200,7 +200,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { dimension.setValue(true); } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.getValue(), stored.getValue(), hasDocValues.getValue(), @@ -211,7 +211,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { indexMode ); return new UnsignedLongFieldMapper( - name(), + leafName(), fieldType, multiFieldsBuilder.build(this, context), copyTo, @@ -679,7 +679,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), ignoreMalformedByDefault, indexMode).dimension(dimension).metric(metricType).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, indexMode).dimension(dimension).metric(metricType).init(this); } /** @@ -777,7 +777,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) { + return new SortedNumericDocValuesSyntheticFieldLoader(name(), leafName(), ignoreMalformed()) { @Override protected void writeValue(XContentBuilder b, long value) throws IOException { b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value)); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index bab91e5d99eca..e1c600edab6fb 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -112,14 +112,14 @@ static class Builder extends FieldMapper.Builder { } private VersionStringFieldType buildFieldType(MapperBuilderContext context, FieldType fieldtype) { - return new VersionStringFieldType(context.buildFullName(name()), fieldtype, meta.getValue()); + return new VersionStringFieldType(context.buildFullName(leafName()), fieldtype, meta.getValue()); } @Override public VersionStringFieldMapper build(MapperBuilderContext context) { FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); return new VersionStringFieldMapper( - name(), + leafName(), fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), @@ -442,7 +442,7 @@ public String toString() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName()).init(this); + return new Builder(leafName()).init(this); } @Override @@ -457,7 +457,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName(), null, false) { + return new SortedSetDocValuesSyntheticFieldLoader(name(), leafName(), null, false) { @Override protected BytesRef convert(BytesRef value) { return VersionEncoder.decodeVersion(value); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index a8f437f476ada..158111bb98de8 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -173,7 +173,7 @@ private FieldValues scriptValues() { GeometryFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), GeometryFieldScript.CONTEXT); return factory == null ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory(name(), script.get().getParams(), lookup, OnScriptError.FAIL) + : (lookup, ctx, doc, consumer) -> factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) .newInstance(ctx) .runForDoc(doc, consumer); } @@ -194,7 +194,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { ); GeoShapeParser parser = new GeoShapeParser(geometryParser, orientation.get().value()); GeoShapeWithDocValuesFieldType ft = new GeoShapeWithDocValuesFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get(), hasDocValues.get(), stored.get(), @@ -206,7 +206,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { ); if (script.get() == null) { return new GeoShapeWithDocValuesFieldMapper( - name(), + leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, @@ -216,7 +216,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { ); } return new GeoShapeWithDocValuesFieldMapper( - name(), + leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, @@ -458,7 +458,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { return new Builder( - simpleName(), + leafName(), builder.version, builder.scriptCompiler, builder.ignoreMalformed.getDefaultValue().value(), diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index 1657a3bf7fbce..de49e0c5a5563 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -105,14 +105,14 @@ public FieldMapper build(MapperBuilderContext context) { ); } CartesianPointParser parser = new CartesianPointParser( - name(), + leafName(), p -> CartesianPoint.parsePoint(p, ignoreZValue.get().value()), nullValue.get(), ignoreZValue.get().value(), ignoreMalformed.get().value() ); PointFieldType ft = new PointFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get(), stored.get(), hasDocValues.get(), @@ -120,7 +120,7 @@ public FieldMapper build(MapperBuilderContext context) { nullValue.get(), meta.get() ); - return new PointFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); + return new PointFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } @@ -177,7 +177,7 @@ public PointFieldType fieldType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), builder.ignoreMalformed.getDefaultValue().value()).init(this); + return new Builder(leafName(), builder.ignoreMalformed.getDefaultValue().value()).init(this); } public static class PointFieldType extends AbstractPointFieldType implements ShapeQueryable { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 83e434f829591..4cc983592d0c1 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -118,14 +118,14 @@ public ShapeFieldMapper build(MapperBuilderContext context) { ); Parser parser = new ShapeParser(geometryParser); ShapeFieldType ft = new ShapeFieldType( - context.buildFullName(name()), + context.buildFullName(leafName()), indexed.get(), hasDocValues.get(), orientation.get().value(), parser, meta.get() ); - return new ShapeFieldMapper(name(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); + return new ShapeFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } @@ -237,7 +237,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { return new Builder( - simpleName(), + leafName(), builder.version, builder.ignoreMalformed.getDefaultValue().value(), builder.coerce.getDefaultValue().value() diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml index fc2e22d857358..b971c246ac50a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml @@ -356,6 +356,7 @@ Field with ignored_malformed: Authorization: "ApiKey ${credentials}" search: index: index_fls + sort: name - match: { hits.hits.0._source.name: A } - is_false: "hits.hits.0._source.secret" - match: { hits.hits.1._source.name: B } diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java index b2dc04c1178e4..4d2789dbb8591 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java @@ -51,7 +51,12 @@ public class LegacyStackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "true"); + private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of( + "xpack.stack.template.deprecated", + "true", + "xpack.stack.template.logs.index.mode", + "standard" + ); // General mappings conventions for any data that ends up in a data stream public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams-mappings"; diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 34cacbb8956e5..aa1e8858163a5 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -68,10 +68,9 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final ClusterService clusterService; private final FeatureService featureService; + private final Map componentTemplateConfigs; private volatile boolean stackTemplateEnabled; - private final boolean logsIndexModeTemplateEnabled; - public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); // General mappings conventions for any data that ends up in a data stream @@ -132,53 +131,10 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); - this.logsIndexModeTemplateEnabled = CLUSTER_LOGSDB_ENABLED.get(nodeSettings); - } - - @Override - public void initialize() { - super.initialize(); - clusterService.getClusterSettings().addSettingsUpdateConsumer(STACK_TEMPLATES_ENABLED, this::updateEnabledSetting); - } - - private void updateEnabledSetting(boolean newValue) { - if (newValue) { - this.stackTemplateEnabled = true; - } else { - logger.info( - "stack composable templates [{}] and component templates [{}] will not be installed or reinstalled", - String.join(",", getComposableTemplateConfigs().keySet()), - String.join(",", getComponentTemplateConfigs().keySet()) - ); - this.stackTemplateEnabled = false; - } - } - - private static final List LIFECYCLE_POLICY_CONFIGS = List.of( - new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(METRICS_ILM_POLICY_NAME, "/metrics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(SYNTHETICS_ILM_POLICY_NAME, "/synthetics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(ILM_7_DAYS_POLICY_NAME, "/7-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(ILM_30_DAYS_POLICY_NAME, "/30-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(ILM_90_DAYS_POLICY_NAME, "/90-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(ILM_180_DAYS_POLICY_NAME, "/180-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), - new LifecyclePolicyConfig(ILM_365_DAYS_POLICY_NAME, "/365-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES) - ); - - @Override - protected List getLifecycleConfigs() { - return LIFECYCLE_POLICY_CONFIGS; - } - - @Override - protected List getLifecyclePolicies() { - return lifecyclePolicies; + this.componentTemplateConfigs = loadComponentTemplateConfigs(CLUSTER_LOGSDB_ENABLED.get(nodeSettings)); } - private static final Map COMPONENT_TEMPLATE_CONFIGS; - private static final Map LOGSDB_COMPONENT_TEMPLATE_CONFIGS; - - static { + private Map loadComponentTemplateConfigs(boolean logsDbEnabled) { final Map componentTemplates = new HashMap<>(); for (IndexTemplateConfig config : List.of( new IndexTemplateConfig( @@ -190,7 +146,7 @@ protected List getLifecyclePolicies() { ), new IndexTemplateConfig( LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/logs@mappings.json", + logsDbEnabled ? "/logs@mappings-logsdb.json" : "/logs@mappings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES @@ -207,7 +163,12 @@ protected List getLifecyclePolicies() { "/logs@settings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES + Map.of( + "xpack.stack.template.deprecated", + "false", + "xpack.stack.template.logs.index.mode", + logsDbEnabled ? "logs" : "standard" + ) ), new IndexTemplateConfig( METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, @@ -261,99 +222,52 @@ protected List getLifecyclePolicies() { throw new AssertionError(e); } } - COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(componentTemplates); + return Map.copyOf(componentTemplates); + } - final Map logsdbComponentTemplates = new HashMap<>(); - for (IndexTemplateConfig config : List.of( - new IndexTemplateConfig( - DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/data-streams@mappings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/logs@mappings-logsdb.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - ECS_DYNAMIC_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/ecs@mappings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/logs@settings-logsdb.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/metrics@mappings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/metrics@settings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - METRICS_TSDB_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/metrics@tsdb-settings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/synthetics@mappings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/synthetics@settings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, - "/kibana-reporting@settings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ) - )) { - try { - logsdbComponentTemplates.put( - config.getTemplateName(), - ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) - ); - } catch (IOException e) { - throw new AssertionError(e); - } + @Override + public void initialize() { + super.initialize(); + clusterService.getClusterSettings().addSettingsUpdateConsumer(STACK_TEMPLATES_ENABLED, this::updateEnabledSetting); + } + + private void updateEnabledSetting(boolean newValue) { + if (newValue) { + this.stackTemplateEnabled = true; + } else { + logger.info( + "stack composable templates [{}] and component templates [{}] will not be installed or reinstalled", + String.join(",", getComposableTemplateConfigs().keySet()), + String.join(",", getComponentTemplateConfigs().keySet()) + ); + this.stackTemplateEnabled = false; } - LOGSDB_COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(logsdbComponentTemplates); + } + + private static final List LIFECYCLE_POLICY_CONFIGS = List.of( + new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(METRICS_ILM_POLICY_NAME, "/metrics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(SYNTHETICS_ILM_POLICY_NAME, "/synthetics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(ILM_7_DAYS_POLICY_NAME, "/7-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(ILM_30_DAYS_POLICY_NAME, "/30-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(ILM_90_DAYS_POLICY_NAME, "/90-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(ILM_180_DAYS_POLICY_NAME, "/180-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES), + new LifecyclePolicyConfig(ILM_365_DAYS_POLICY_NAME, "/365-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES) + ); + + @Override + protected List getLifecycleConfigs() { + return LIFECYCLE_POLICY_CONFIGS; + } + + @Override + protected List getLifecyclePolicies() { + return lifecyclePolicies; } @Override protected Map getComponentTemplateConfigs() { - if (logsIndexModeTemplateEnabled) { - return LOGSDB_COMPONENT_TEMPLATE_CONFIGS; - } - return COMPONENT_TEMPLATE_CONFIGS; + return componentTemplateConfigs; } private static final Map COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates( diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 6f2c15836a04a..8a895bb3627ce 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -238,8 +238,14 @@ Builder nullValue(String nullValue) { @Override public WildcardFieldMapper build(MapperBuilderContext context) { return new WildcardFieldMapper( - name(), - new WildcardFieldType(context.buildFullName(name()), nullValue.get(), ignoreAbove.get(), indexVersionCreated, meta.get()), + leafName(), + new WildcardFieldType( + context.buildFullName(leafName()), + nullValue.get(), + ignoreAbove.get(), + indexVersionCreated, + meta.get() + ), ignoreAbove.get(), context.isSourceSynthetic(), multiFieldsBuilder.build(this, context), @@ -982,7 +988,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexVersionCreated).init(this); + return new Builder(leafName(), indexVersionCreated).init(this); } @Override @@ -1047,10 +1053,10 @@ public void write(XContentBuilder b) throws IOException { case 0: return; case 1: - b.field(simpleName()); + b.field(leafName()); break; default: - b.startArray(simpleName()); + b.startArray(leafName()); } for (int i = 0; i < docValueCount; i++) { int length = docValuesStream.readVInt();