Merge branch 'main' into ES-9378

docs/changelog/116423.yaml

@@ -0,0 +1,5 @@
+pr: 116423
+summary: Support mTLS for the Elastic Inference Service integration inside the inference API
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/118173.yaml

@@ -0,0 +1,5 @@
+pr: 118173
+summary: ES|QL categorize with multiple groupings
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/118194.yaml

@@ -0,0 +1,5 @@
+pr: 118194
+summary: Retry on `ClusterBlockException` on transform destination index
+area: Machine Learning
+type: enhancement
+issues: []

docs/reference/mapping/params/index-prefixes.asciidoc

@@ -54,3 +54,30 @@ PUT my-index-000001
   }
 }
 --------------------------------
+
+`index_prefixes` parameter instructs {ES} to create a subfield "._index_prefix". This
+field will be used to do fast prefix queries. When doing highlighting, add "._index_prefix"
+subfield to the `matched_fields` parameter to highlight the main field based on the
+found matches of the prefix field, like in the request below:
+
+[source,console]
+--------------------------------
+GET my-index-000001/_search
+{
+  "query": {
+    "prefix": {
+      "full_name": {
+        "value": "ki"
+      }
+    }
+  },
+  "highlight": {
+    "fields": {
+      "full_name": {
+        "matched_fields": ["full_name._index_prefix"]
+      }
+    }
+  }
+}
+--------------------------------
+// TEST[continued]

docs/reference/mapping/types/search-as-you-type.asciidoc

@@ -97,11 +97,21 @@ GET my-index-000001/_search
         "my_field._3gram"
       ]
     }
+  },
+  "highlight": {
+    "fields": {
+      "my_field": {
+        "matched_fields": ["my_field._index_prefix"] <1>
+      }
+    }
   }
 }
 --------------------------------------------------
 // TEST[continued]
 
+<1> Adding "my_field._index_prefix" to the `matched_fields` allows to highlight
+    "my_field" also based on matches from "my_field._index_prefix" field.
+
 [source,console-result]
 --------------------------------------------------
 {
@@ -126,6 +136,11 @@ GET my-index-000001/_search
         "_score" : 0.8630463,
         "_source" : {
           "my_field" : "quick brown fox jump lazy dog"
+        },
+        "highlight": {
+          "my_field": [
+            "quick <em>brown fox jump lazy</em> dog"
+          ]
         }
       }
     ]

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml

@@ -84,32 +84,6 @@
   - match: { error.type: "illegal_argument_exception" }
   - match: { error.reason: "Types cannot be provided in put mapping requests" }
 
----
-"Put mappings with explicit _doc type bwc":
-  - skip:
-          cluster_features: [ "gte_v8.0.0"]
-          reason:      "old deprecation message for pre 8.0"
-  - requires:
-          test_runner_features: ["node_selector"]
-  - do:
-      indices.create:
-        index: test_index
-
-  - do:
-      node_selector:
-        version: "original"
-      catch: bad_request
-      indices.put_mapping:
-        index: test_index
-        body:
-          _doc:
-            properties:
-              field:
-                type: keyword
-
-  - match: { error.type: "illegal_argument_exception" }
-  - match: { error.reason: "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true." }
-
 ---
 "Update per-field metadata":
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -1092,10 +1092,10 @@ public IndexVersion getCreationVersion() {
 
     /**
      * Return the {@link IndexVersion} that this index provides compatibility for.
-     * This is typically compared to the {@link IndexVersions#MINIMUM_COMPATIBLE} to figure out whether the index can be handled
-     * by the cluster.
-     * By default, this is equal to the {@link #getCreationVersion()}, but can also be a newer version if the index has been imported as
-     * a legacy index from an older snapshot, and its metadata has been converted to be handled by newer version nodes.
+     * This is typically compared to the {@link IndexVersions#MINIMUM_COMPATIBLE} or {@link IndexVersions#MINIMUM_READONLY_COMPATIBLE}
+     * to figure out whether the index can be handled by the cluster.
+     * By default, this is equal to the {@link #getCreationVersion()}, but can also be a newer version if the index has been created by
+     * a legacy version, and imported archive, in which case its metadata has been converted to be handled by newer version nodes.
      */
     public IndexVersion getCompatibilityVersion() {
         return indexCompatibilityVersion;

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -523,8 +523,7 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr
 
         logger.info("oldest index version recorded in NodeMetadata {}", metadata.oldestIndexVersion());
 
-        if (metadata.oldestIndexVersion().isLegacyIndexVersion()) {
-
+        if (metadata.oldestIndexVersion().before(IndexVersions.MINIMUM_COMPATIBLE)) {
             BuildVersion bestDowngradeVersion = getBestDowngradeVersion(metadata.previousNodeVersion());
             throw new IllegalStateException(
                 "Cannot start this node because it holds metadata for indices with version ["

server/src/main/java/org/elasticsearch/index/IndexVersion.java

@@ -123,8 +123,13 @@ public static IndexVersion current() {
         return CurrentHolder.CURRENT;
     }
 
+    /**
+     * Returns whether this index version is supported by this node version out-of-the-box.
+     * This is used to distinguish between ordinary indices and archive indices that may be
+     * imported into the cluster in read-only mode, and with limited functionality.
+     */
     public boolean isLegacyIndexVersion() {
-        return before(IndexVersions.MINIMUM_COMPATIBLE);
+        return before(IndexVersions.MINIMUM_READONLY_COMPATIBLE);
     }
 
     public static IndexVersion getMinimumCompatibleIndexVersion(int versionId) {

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -185,6 +185,7 @@ private static Version parseUnchecked(String version) {
      */
 
     public static final IndexVersion MINIMUM_COMPATIBLE = V_8_0_0;
+    public static final IndexVersion MINIMUM_READONLY_COMPATIBLE = V_7_0_0;
 
     static final NavigableMap<Integer, IndexVersion> VERSION_IDS = getAllVersionIds(IndexVersions.class);
     static final IndexVersion LATEST_DEFINED;
@@ -200,7 +201,7 @@ static NavigableMap<Integer, IndexVersion> getAllVersionIds(Class<?> cls) {
         Map<Integer, String> versionIdFields = new HashMap<>();
         NavigableMap<Integer, IndexVersion> builder = new TreeMap<>();
 
-        Set<String> ignore = Set.of("ZERO", "MINIMUM_COMPATIBLE");
+        Set<String> ignore = Set.of("ZERO", "MINIMUM_COMPATIBLE", "MINIMUM_READONLY_COMPATIBLE");
 
         for (Field declaredField : cls.getFields()) {
             if (declaredField.getType().equals(IndexVersion.class)) {

server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java

@@ -102,7 +102,9 @@ public RepositoriesModule(
         }
         if (preRestoreChecks.isEmpty()) {
             preRestoreChecks.add((snapshot, version) -> {
-                if (version.isLegacyIndexVersion()) {
+                // pre-restore checks will be run against the version in which the snapshot was created as well as
+                // the version in which the restored index was created
+                if (version.before(IndexVersions.MINIMUM_COMPATIBLE)) {
                     throw new SnapshotRestoreException(
                         snapshot,
                         "the snapshot was created with Elasticsearch version ["

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java

@@ -78,10 +78,7 @@ static Map<String, Object> prepareMappings(Map<String, Object> source) {
         Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
         if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) {
             throw new IllegalArgumentException(
-                "The mapping definition cannot be nested under a type "
-                    + "["
-                    + MapperService.SINGLE_MAPPING_NAME
-                    + "] unless include_type_name is set to true."
+                "The mapping definition cannot be nested under a type [" + MapperService.SINGLE_MAPPING_NAME + "]."
             );
         }
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java

@@ -9,15 +9,12 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -35,13 +32,6 @@
 
 public class RestGetFieldMappingAction extends BaseRestHandler {
 
-    private static final Logger logger = LogManager.getLogger(RestGetFieldMappingAction.class);
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName());
-    public static final String INCLUDE_TYPE_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get "
-        + "field mapping requests is deprecated. The parameter will be removed in the next major version.";
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get field mapping request is deprecated. "
-        + "Use typeless api instead";
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_mapping/field/{fields}"), new Route(GET, "/{index}/_mapping/field/{fields}"));

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -35,9 +34,6 @@
  */
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetIndicesAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndicesAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests"
-        + " is deprecated. The parameter will be removed in the next major version.";
 
     @Override
     public List<Route> routes() {

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -31,11 +30,6 @@
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetMappingAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingAction.class);
-    public static final String INCLUDE_TYPE_DEPRECATION_MSG = "[types removal] Using include_type_name in get"
-        + " mapping requests is deprecated. The parameter will be removed in the next major version.";
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get mapping request is deprecated. "
-        + "Use typeless api instead";
 
     public RestGetMappingAction() {}
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -27,7 +26,6 @@
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 public abstract class RestResizeHandler extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestResizeHandler.class);
 
     RestResizeHandler() {}
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -33,10 +32,6 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestRolloverIndexAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestRolloverIndexAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in rollover "
-        + "index requests is deprecated. The parameter will be removed in the next major version.";
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(POST, "/{index}/_rollover"), new Route(POST, "/{index}/_rollover/{new_index}"));

server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java

@@ -12,24 +12,16 @@
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.hasKey;
-
 public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase<GetFieldMappingsResponse> {
 
     public void testManualSerialization() throws IOException {
@@ -56,36 +48,6 @@ public void testNullFieldMappingToXContent() {
         assertEquals("{\"index\":{\"mappings\":{}}}", Strings.toString(response));
     }
 
-    public void testToXContentIncludesType() throws Exception {
-        Map<String, Map<String, FieldMappingMetadata>> mappings = new HashMap<>();
-        FieldMappingMetadata fieldMappingMetadata = new FieldMappingMetadata("my field", new BytesArray("{}"));
-        mappings.put("index", Collections.singletonMap("field", fieldMappingMetadata));
-        GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings);
-        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("include_type_name", "true"));
-
-        // v8 does not have _doc, even when include_type_name is present
-        // (although this throws unconsumed parameter exception in RestGetFieldMappingsAction)
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) {
-            response.toXContent(builder, params);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                @SuppressWarnings("unchecked")
-                Map<String, Map<String, Object>> index = (Map<String, Map<String, Object>>) parser.map().get("index");
-                assertThat(index.get("mappings"), hasKey("field"));
-            }
-        }
-
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) {
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                @SuppressWarnings("unchecked")
-                Map<String, Map<String, Object>> index = (Map<String, Map<String, Object>>) parser.map().get("index");
-                assertThat(index.get("mappings"), hasKey("field"));
-            }
-        }
-    }
-
     @Override
     protected GetFieldMappingsResponse createTestInstance() {
         return new GetFieldMappingsResponse(randomMapping());

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AllocateAction.java

@@ -20,8 +20,6 @@
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -73,17 +71,17 @@ public AllocateAction(
         Map<String, String> require
     ) {
         if (include == null) {
-            this.include = Collections.emptyMap();
+            this.include = Map.of();
         } else {
             this.include = include;
         }
         if (exclude == null) {
-            this.exclude = Collections.emptyMap();
+            this.exclude = Map.of();
         } else {
             this.exclude = exclude;
         }
         if (require == null) {
-            this.require = Collections.emptyMap();
+            this.require = Map.of();
         } else {
             this.require = require;
         }
@@ -201,7 +199,7 @@ public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
         }
         UpdateSettingsStep allocateStep = new UpdateSettingsStep(allocateKey, allocationRoutedKey, client, newSettings.build());
         AllocationRoutedStep routedCheckStep = new AllocationRoutedStep(allocationRoutedKey, nextStepKey);
-        return Arrays.asList(allocateStep, routedCheckStep);
+        return List.of(allocateStep, routedCheckStep);
     }
 
     @Override