Merge branch '8.x' into backport/8.x/pr-115031

piergm · Oct 21, 2024 · 3379e3c · 3379e3c
2 parents a3c2d69 + ade7f7c
commit 3379e3c
Show file tree

Hide file tree

Showing 99 changed files with 4,094 additions and 547 deletions.
diff --git a/.../elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java b/.../elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java
@@ -21,7 +21,7 @@ public abstract class AbstractDocValuesForUtilBenchmark {
     protected final int blockSize;
 
     public AbstractDocValuesForUtilBenchmark() {
-        this.forUtil = new DocValuesForUtil();
+        this.forUtil = new DocValuesForUtil(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
         this.blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
     }
 

diff --git a/docs/changelog/115048.yaml b/docs/changelog/115048.yaml
@@ -0,0 +1,5 @@
+pr: 115048
+summary: Add timeout and cancellation check to rescore phase
+area: Ranking
+type: enhancement
+issues: []
diff --git a/docs/changelog/115147.yaml b/docs/changelog/115147.yaml
@@ -0,0 +1,5 @@
+pr: 115147
+summary: Fix IPinfo geolocation schema
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc
@@ -155,11 +155,6 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`annotated_text` fields support {ref}/mapping-source-field.html#synthetic-source[synthetic `_source`] if they have
-a {ref}/keyword.html#keyword-synthetic-source[`keyword`] sub-field that supports synthetic
-`_source` or if the `annotated_text` field sets `store` to `true`. Either way, it may
-not have {ref}/copy-to.html[`copy_to`].
-
 If using a sub-`keyword` field then the values are sorted in the same way as
 a `keyword` field's values are sorted. By default, that means sorted with
 duplicates removed. So:

diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc
@@ -59,8 +59,7 @@ since it relies on separate data structures to perform the search. Before
 using the <<include-exclude, `excludes`>> parameter, make sure to review the
 downsides of omitting fields from `_source`.
 
-Another option is to use  <<synthetic-source,synthetic `_source`>> if all
-your index fields support it.
+Another option is to use  <<synthetic-source,synthetic `_source`>>.
 
 [discrete]
 === Ensure data nodes have enough memory

diff --git a/docs/reference/inference/images/inference-landscape.jpg b/docs/reference/inference/images/inference-landscape.jpg
diff --git a/docs/reference/inference/images/inference-landscape.png b/docs/reference/inference/images/inference-landscape.png
diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
@@ -25,7 +25,7 @@ the following APIs to manage {infer} models and perform {infer}:
 
 [[inference-landscape]]
 .A representation of the Elastic inference landscape
-image::images/inference-landscape.png[A representation of the Elastic inference landscape,align="center"]
+image::images/inference-landscape.jpg[A representation of the Elastic inference landscape,align="center"]
 
 An {infer} endpoint enables you to use the corresponding {ml} model without
 manual deployment and apply it to your data at ingestion time through

diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc
@@ -259,9 +259,6 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`aggregate_metric-double` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration.
-
 For example:
 [source,console,id=synthetic-source-aggregate-metric-double-example]
 ----

diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
@@ -63,7 +63,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`binary` fields support <<synthetic-source,synthetic `_source`>> only when <<doc-values,`doc_values`>> are enabled. Synthetic source always sorts `binary` values in order of their byte representation. For example:
+Synthetic source may sort `binary` values in order of their byte representation. For example:
 [source,console,id=synthetic-source-binary-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
@@ -241,10 +241,9 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `boolean` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
+default configuration.
 
-Synthetic source always sorts `boolean` fields. For example:
+Synthetic source may sort `boolean` field values. For example:
 [source,console,id=synthetic-source-boolean-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
@@ -247,11 +247,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`date` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `date` fields. For example:
+Synthetic source may sort `date` field values. For example:
 [source,console,id=synthetic-source-date-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc
@@ -150,12 +150,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`date_nanos` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>, <<ignore-malformed,`ignore_malformed`>> set to true
-or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `date_nanos` fields. For example:
+Synthetic source may sort `date_nanos` field values. For example:
 [source,console,id=synthetic-source-date-nanos-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc
@@ -325,10 +325,9 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 Flattened fields support <<synthetic-source,synthetic`_source`>> in their default
-configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>>
-disabled.
+configuration.
 
-Synthetic source always sorts alphabetically and de-duplicates flattened fields.
+Synthetic source may sort `flattened` field values and remove duplicates.
 For example:
 [source,console,id=synthetic-source-flattened-sorting-example]
 ----

diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
@@ -219,11 +219,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`geo_point` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>> or with
-<<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `geo_point` fields (first by latitude and then
+Synthetic source may sort `geo_point` fields (first by latitude and then
 longitude) and reduces them to their stored precision. For example:
 [source,console,id=synthetic-source-geo-point-example]
 ----

diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -502,6 +502,3 @@ synthetic `_source` is in technical preview. Features in technical preview may
 be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
-
-`geo_shape` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration.
diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc
@@ -79,7 +79,7 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `histogram` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with <<copy-to,`copy_to`>>.
+default configuration.
 
 NOTE: To save space, zero-count buckets are not stored in the histogram doc values.
 As a result, when indexing a histogram field in an index with synthetic source enabled,

diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
@@ -161,11 +161,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`ip` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
-
-Synthetic source always sorts `ip` fields and removes duplicates. For example:
+Synthetic source may sort `ip` field values and remove duplicates. For example:
 [source,console,id=synthetic-source-ip-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
@@ -178,11 +178,7 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`keyword` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-a <<normalizer,`normalizer`>> or <<copy-to,`copy_to`>>.
-
-By default, synthetic source sorts `keyword` fields and removes duplicates.
+Synthetic source may sort `keyword` fields and remove duplicates.
 For example:
 [source,console,id=synthetic-source-keyword-example-default]
 ----

diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
@@ -254,7 +254,7 @@ All numeric fields support <<synthetic-source,synthetic
 together with <<copy-to,`copy_to`>>, or
 with <<doc-values,`doc_values`>> disabled.
 
-Synthetic source always sorts numeric fields. For example:
+Synthetic source may sort numeric field values. For example:
 [source,console,id=synthetic-source-numeric-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc
@@ -247,9 +247,9 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `range` fields support <<synthetic-source,synthetic `_source`>> in their default
-configuration. Synthetic `_source` cannot be used with <<doc-values,`doc_values`>> disabled.
+configuration.
 
-Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example:
+Synthetic source may sort `range` field values and remove duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example:
 [source,console,id=synthetic-source-range-sorting-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -266,5 +266,4 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `search_as_you_type` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>.
+default configuration.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
@@ -134,10 +134,6 @@ The following parameters are accepted by `text` fields:
 
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false` (default).
-    This parameter will be automatically set to `true` for TSDB indices
-    (indices that have `index.mode` set to `time_series`)
-    if there is no <<keyword-synthetic-source, `keyword`>>
-    sub-field that supports synthetic `_source`.
 
 <<search-analyzer,`search_analyzer`>>::
 

diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
@@ -103,5 +103,4 @@ any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
 `token_count` fields support <<synthetic-source,synthetic `_source`>> in their
-default configuration. Synthetic `_source` cannot be used together with
-<<copy-to,`copy_to`>>.
+default configuration.
diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc
@@ -77,10 +77,10 @@ be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
 of official GA features.
 
-`version` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
-declare <<copy-to,`copy_to`>>.
+`version` fields support <<synthetic-source,synthetic `_source`>> in their
+default configuration..
 
-Synthetic source always sorts `version` fields and removes duplicates. For example:
+Synthetic source may sort `version` field values and remove duplicates. For example:
 [source,console,id=synthetic-source-version-example]
 ----
 PUT idx

diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc
@@ -133,10 +133,8 @@ The following parameters are accepted by `wildcard` fields:
 
 [[wildcard-synthetic-source]]
 ==== Synthetic `_source`
-`wildcard` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
-declare <<copy-to,`copy_to`>>.
 
-Synthetic source always sorts `wildcard` fields. For example:
+Synthetic source may sort `wildcard` field values. For example:
 [source,console,id=synthetic-source-wildcard-example]
 ----
 PUT idx

diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
@@ -17,7 +17,7 @@ commons-codec = "commons-codec:commons-codec:1.11"
 commmons-io = "commons-io:commons-io:2.2"
 docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5"
 forbiddenApis = "de.thetaphi:forbiddenapis:3.6"
-gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.17.4"
+gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.18.1"
 hamcrest = "org.hamcrest:hamcrest:2.1"
 httpcore = "org.apache.httpcomponents:httpcore:4.4.12"
 httpclient = "org.apache.httpcomponents:httpclient:4.5.14"

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -799,6 +799,11 @@
             <sha256 value="e2b3f8a191b0b401b75c2c4542d3d1719814a4212e6920fae4f2f940678bfd99" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.gradle" name="develocity-gradle-plugin" version="3.18.1">
+         <artifact name="develocity-gradle-plugin-3.18.1.jar">
+            <sha256 value="0adf32afef4ae46a131367e393d18a5d9529750dfd1432287797a64b2c36c7ef" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.h2database" name="h2" version="1.4.197">
          <artifact name="h2-1.4.197.jar">
             <sha256 value="37f5216e14af2772930dff9b8734353f0a80e89ba3f33e065441de6537c5e842" origin="Generated by Gradle"/>

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java
@@ -218,8 +218,8 @@ public record CountryResult(
     public record GeolocationResult(
         String city,
         String country,
-        Double latitude,
-        Double longitude,
+        Double lat,
+        Double lng,
         String postalCode,
         String region,
         String timezone
@@ -229,14 +229,15 @@ public record GeolocationResult(
         public GeolocationResult(
             @MaxMindDbParameter(name = "city") String city,
             @MaxMindDbParameter(name = "country") String country,
-            @MaxMindDbParameter(name = "latitude") String latitude,
-            @MaxMindDbParameter(name = "longitude") String longitude,
-            // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
+            // @MaxMindDbParameter(name = "geoname_id") String geonameId, // for now we're not exposing this
+            @MaxMindDbParameter(name = "lat") String lat,
+            @MaxMindDbParameter(name = "lng") String lng,
             @MaxMindDbParameter(name = "postal_code") String postalCode,
             @MaxMindDbParameter(name = "region") String region,
+            // @MaxMindDbParameter(name = "region_code") String regionCode, // for now we're not exposing this
             @MaxMindDbParameter(name = "timezone") String timezone
         ) {
-            this(city, country, parseLocationDouble(latitude), parseLocationDouble(longitude), postalCode, region, timezone);
+            this(city, country, parseLocationDouble(lat), parseLocationDouble(lng), postalCode, region, timezone);
         }
     }
 
@@ -395,8 +396,8 @@ protected Map<String, Object> transform(final Result<GeolocationResult> result)
                         }
                     }
                     case LOCATION -> {
-                        Double latitude = response.latitude;
-                        Double longitude = response.longitude;
+                        Double latitude = response.lat;
+                        Double longitude = response.lng;
                         if (latitude != null && longitude != null) {
                             Map<String, Object> locationObject = new HashMap<>();
                             locationObject.put("lat", latitude);

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -82,13 +82,13 @@ public void testMaxmindCity() throws Exception {
     }
 
     public void testIpinfoGeolocation() throws Exception {
-        String ip = "13.107.39.238";
+        String ip = "72.20.12.220";
         GeoIpProcessor processor = new GeoIpProcessor(
             IP_LOCATION_TYPE, // n.b. this is an "ip_location" processor
             randomAlphaOfLength(10),
             null,
             "source_field",
-            loader("ipinfo/ip_geolocation_sample.mmdb"),
+            loader("ipinfo/ip_geolocation_standard_sample.mmdb"),
             () -> true,
             "target_field",
             getIpinfoGeolocationLookup(),
@@ -107,7 +107,7 @@ public void testIpinfoGeolocation() throws Exception {
         Map<String, Object> data = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(data, notNullValue());
         assertThat(data.get("ip"), equalTo(ip));
-        assertThat(data.get("city_name"), equalTo("Des Moines"));
+        assertThat(data.get("city_name"), equalTo("Chicago"));
         // see IpinfoIpDataLookupsTests for more tests of the data lookup behavior
     }
 

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java
@@ -47,6 +47,7 @@ public static void startServer() throws Throwable {
         server.createContext("/404/", exchange -> {
             try {
                 exchange.sendResponseHeaders(404, 0);
+                exchange.close();
             } catch (Exception e) {
                 fail(e);
             }
@@ -102,6 +103,7 @@ public boolean checkCredentials(String username, String password) {
                     exchange.getResponseHeaders().add("Location", "/" + destination + "/");
                 }
                 exchange.sendResponseHeaders(302, 0);
+                exchange.close();
             } catch (Exception e) {
                 fail(e);
             }