Merge branch 'main' into test/old_codecs_N-2

docs/changelog/118103.yaml

@@ -0,0 +1,11 @@
+pr: 118103
+summary: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7"
+area: Infra/Core
+type: breaking
+issues: []
+breaking:
+  title: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7"
+  area: REST API
+  details: "This PR removes all references to V_7 in the Rest API. V7 features marked for deprecation have been removed."
+  impact: "This change is breaking for any external plugins/clients that rely on the V_7 enum or deprecated version 7 functionality"
+  notable: false

docs/changelog/118435.yaml

@@ -0,0 +1,6 @@
+pr: 118435
+summary: '`_score` should not be a reserved attribute in ES|QL'
+area: ES|QL
+type: enhancement
+issues:
+ - 118460

libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java

@@ -20,10 +20,7 @@ public enum RestApiVersion {
 
     V_9(9),
 
-    V_8(8),
-
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove all references to V_7 then delete this annotation
-    V_7(7);
+    V_8(8);
 
     public final byte major;
 
@@ -54,23 +51,18 @@ public static Predicate<RestApiVersion> equalTo(RestApiVersion restApiVersion) {
         return switch (restApiVersion) {
             case V_9 -> r -> r.major == V_9.major;
             case V_8 -> r -> r.major == V_8.major;
-            case V_7 -> r -> r.major == V_7.major;
         };
     }
 
     public static Predicate<RestApiVersion> onOrAfter(RestApiVersion restApiVersion) {
         return switch (restApiVersion) {
             case V_9 -> r -> r.major >= V_9.major;
             case V_8 -> r -> r.major >= V_8.major;
-            case V_7 -> r -> r.major >= V_7.major;
         };
     }
 
     public static RestApiVersion forMajor(int major) {
         switch (major) {
-            case 7 -> {
-                return V_7;
-            }
             case 8 -> {
                 return V_8;
             }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -18,7 +18,6 @@
 import java.lang.module.ModuleFinder;
 import java.lang.module.ModuleReference;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
@@ -56,8 +55,8 @@ public <E extends Entitlement> Stream<E> getEntitlements(Class<E> entitlementCla
 
     final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new HashMap<>();
 
-    protected final Policy serverPolicy;
-    protected final Map<String, Policy> pluginPolicies;
+    protected final Map<String, List<Entitlement>> serverEntitlements;
+    protected final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, String> pluginResolver;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
@@ -79,19 +78,16 @@ private static Set<Module> findSystemModules() {
     }
 
     public PolicyManager(Policy defaultPolicy, Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
-        this.serverPolicy = Objects.requireNonNull(defaultPolicy);
-        this.pluginPolicies = Collections.unmodifiableMap(Objects.requireNonNull(pluginPolicies));
+        this.serverEntitlements = buildScopeEntitlementsMap(Objects.requireNonNull(defaultPolicy));
+        this.pluginsEntitlements = Objects.requireNonNull(pluginPolicies)
+            .entrySet()
+            .stream()
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue())));
         this.pluginResolver = pluginResolver;
     }
 
-    private static List<Entitlement> lookupEntitlementsForModule(Policy policy, String moduleName) {
-        for (int i = 0; i < policy.scopes.size(); ++i) {
-            var scope = policy.scopes.get(i);
-            if (scope.name.equals(moduleName)) {
-                return scope.entitlements;
-            }
-        }
-        return null;
+    private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy policy) {
+        return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements));
     }
 
     public void checkExitVM(Class<?> callerClass) {
@@ -141,21 +137,21 @@ ModuleEntitlements getEntitlementsOrThrow(Class<?> callerClass, Module requestin
 
         if (isServerModule(requestingModule)) {
             var scopeName = requestingModule.getName();
-            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverPolicy, scopeName);
+            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName);
         }
 
         // plugins
         var pluginName = pluginResolver.apply(callerClass);
         if (pluginName != null) {
-            var pluginPolicy = pluginPolicies.get(pluginName);
-            if (pluginPolicy != null) {
+            var pluginEntitlements = pluginsEntitlements.get(pluginName);
+            if (pluginEntitlements != null) {
                 final String scopeName;
                 if (requestingModule.isNamed() == false) {
                     scopeName = ALL_UNNAMED;
                 } else {
                     scopeName = requestingModule.getName();
                 }
-                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginPolicy, scopeName);
+                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName);
             }
         }
 
@@ -167,15 +163,20 @@ private static String buildModuleNoPolicyMessage(Class<?> callerClass, Module re
         return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName());
     }
 
-    private ModuleEntitlements getModuleEntitlementsOrThrow(Class<?> callerClass, Module module, Policy policy, String moduleName) {
-        var entitlements = lookupEntitlementsForModule(policy, moduleName);
+    private ModuleEntitlements getModuleEntitlementsOrThrow(
+        Class<?> callerClass,
+        Module module,
+        Map<String, List<Entitlement>> scopeEntitlements,
+        String moduleName
+    ) {
+        var entitlements = scopeEntitlements.get(moduleName);
         if (entitlements == null) {
             // Module without entitlements - remember we don't have any
             moduleEntitlementsMap.put(module, ModuleEntitlements.NONE);
             throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module));
         }
         // We have a policy for this module
-        var classEntitlements = createClassEntitlements(entitlements);
+        var classEntitlements = new ModuleEntitlements(entitlements);
         moduleEntitlementsMap.put(module, classEntitlements);
         return classEntitlements;
     }
@@ -184,10 +185,6 @@ private static boolean isServerModule(Module requestingModule) {
         return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot();
     }
 
-    private ModuleEntitlements createClassEntitlements(List<Entitlement> entitlements) {
-        return new ModuleEntitlements(entitlements);
-    }
-
     private static Module requestingModule(Class<?> callerClass) {
         if (callerClass != null) {
             Module callerModule = callerClass.getModule();
@@ -222,6 +219,6 @@ private static boolean isTriviallyAllowed(Module requestingModule) {
 
     @Override
     public String toString() {
-        return "PolicyManager{" + "serverPolicy=" + serverPolicy + ", pluginPolicies=" + pluginPolicies + '}';
+        return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}';
     }
 }

muted-tests.yml

@@ -180,6 +180,12 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT"
   method: "test {scoring.*}"
   issue: https://github.com/elastic/elasticsearch/issues/117641
+- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {scoring.QstrWithFieldAndScoringSortedEval}
   issue: https://github.com/elastic/elasticsearch/issues/117751
@@ -314,6 +320,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/118220
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   issue: https://github.com/elastic/elasticsearch/issues/118238
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test011SecurityEnabledStatus
+  issue: https://github.com/elastic/elasticsearch/issues/118517
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

@@ -177,7 +177,7 @@ private static List<IngestDocument> parseDocs(Map<String, Object> config, RestAp
             String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.INDEX.getFieldName(), "_index");
             String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.ID.getFieldName(), "_id");
             String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, dataMap, Metadata.ROUTING.getFieldName());
-            if (restApiVersion != RestApiVersion.V_8 && dataMap.containsKey(Metadata.TYPE.getFieldName())) {
+            if (dataMap.containsKey(Metadata.TYPE.getFieldName())) {
                 deprecationLogger.compatibleCritical(
                     "simulate_pipeline_with_types",
                     "[types removal] specifying _type in pipeline simulation requests is deprecated"

server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java

@@ -49,7 +49,6 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) {
      */
     default Iterator<? extends ToXContent> toXContentChunked(RestApiVersion restApiVersion, ToXContent.Params params) {
         return switch (restApiVersion) {
-            case V_7 -> throw new AssertionError("v7 API not supported");
             case V_8 -> toXContentChunkedV8(params);
             case V_9 -> toXContentChunked(params);
         };

server/src/main/java/org/elasticsearch/ingest/IngestDocument.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
@@ -957,6 +958,8 @@ void resetTerminate() {
         terminate = false;
     }
 
+    // Unconditionally deprecate the _type field once V7 BWC support is removed
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     public enum Metadata {
         INDEX(IndexFieldMapper.NAME),
         TYPE("_type"),

server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java

@@ -30,7 +30,7 @@ public class BulkRequestParserTests extends ESTestCase {
 
     @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) // Replace with just RestApiVersion.values() when V8 no longer exists
     public static final List<RestApiVersion> REST_API_VERSIONS_POST_V8 = Stream.of(RestApiVersion.values())
-        .filter(v -> v.compareTo(RestApiVersion.V_8) > 0)
+        .filter(v -> v.matches(RestApiVersion.onOrAfter(RestApiVersion.V_9)))
         .toList();
 
     public void testParserCannotBeReusedAfterFailure() {

server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java

@@ -342,7 +342,7 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception {
             requestContent,
             false,
             ingestService,
-            RestApiVersion.V_7
+            RestApiVersion.V_8
         );
         assertThat(actualRequest.verbose(), equalTo(false));
         assertThat(actualRequest.documents().size(), equalTo(numDocs));

server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java

@@ -11,6 +11,7 @@
 
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterIndexInput;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -267,17 +268,47 @@ public void testSkipBytes() throws Exception {
             skipBytesExpected
         );
 
-        IndexInput input = dir.openInput("test", IOContext.DEFAULT);
-        InputStreamIndexInput is = new InputStreamIndexInput(input, limit);
+        var countingInput = new CountingReadBytesIndexInput("test", dir.openInput("test", IOContext.DEFAULT));
+        InputStreamIndexInput is = new InputStreamIndexInput(countingInput, limit);
         is.readNBytes(initialReadBytes);
         assertThat(is.skip(skipBytes), equalTo((long) skipBytesExpected));
+        long expectedActualInitialBytesRead = Math.min(Math.min(initialReadBytes, limit), bytes);
+        assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead));
 
         int remainingBytes = Math.min(bytes, limit) - seekExpected;
         for (int i = seekExpected; i < seekExpected + remainingBytes; i++) {
             assertThat(is.read(), equalTo(i));
         }
+        assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead + remainingBytes));
     }
 
+    protected static class CountingReadBytesIndexInput extends FilterIndexInput {
+        private long bytesRead = 0;
+
+        public CountingReadBytesIndexInput(String resourceDescription, IndexInput in) {
+            super(resourceDescription, in);
+        }
+
+        @Override
+        public byte readByte() throws IOException {
+            long filePointerBefore = getFilePointer();
+            byte b = super.readByte();
+            bytesRead += getFilePointer() - filePointerBefore;
+            return b;
+        }
+
+        @Override
+        public void readBytes(byte[] b, int offset, int len) throws IOException {
+            long filePointerBefore = getFilePointer();
+            super.readBytes(b, offset, len);
+            bytesRead += getFilePointer() - filePointerBefore;
+        }
+
+        public long getBytesRead() {
+            return bytesRead;
+        }
+    };
+
     public void testReadZeroShouldReturnZero() throws IOException {
         try (Directory dir = new ByteBuffersDirectory()) {
             try (IndexOutput output = dir.createOutput("test", IOContext.DEFAULT)) {

x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -142,7 +141,7 @@ static boolean isEnterprise(String typeName) {
      */
     public static final String LICENSE_VERSION_MODE = "license_version";
     /**
-     * Set for {@link RestApiVersion#V_7} requests only
+     * Set for RestApiVersion#V_7 requests only
      * XContent param name to map the "enterprise" license type to "platinum"
      * for backwards compatibility with older clients
      */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java

@@ -22,9 +22,6 @@
 
 public final class MachineLearningField {
 
-    public static final String DEPRECATED_ALLOW_NO_JOBS_PARAM = "allow_no_jobs";
-    public static final String DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM = "allow_no_datafeeds";
-
     public static final Setting<Boolean> AUTODETECT_PROCESS = Setting.boolSetting(
         "xpack.ml.autodetect_process",
         true,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java

@@ -12,9 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -27,10 +25,6 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.core.RestApiVersion.equalTo;
-import static org.elasticsearch.core.RestApiVersion.onOrAfter;
-import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM;
-
 public class CloseJobAction extends ActionType<CloseJobAction.Response> {
 
     public static final CloseJobAction INSTANCE = new CloseJobAction();
@@ -45,11 +39,7 @@ public static class Request extends BaseTasksRequest<Request> implements ToXCont
         public static final ParseField TIMEOUT = new ParseField("timeout");
         public static final ParseField FORCE = new ParseField("force");
 
-        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion
-        public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8));
-        @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7
-        public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_JOBS_PARAM)
-            .forRestApiVersion(equalTo(RestApiVersion.V_7));
+        public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
         public static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
 
         static {
@@ -60,7 +50,6 @@ public static class Request extends BaseTasksRequest<Request> implements ToXCont
             );
             PARSER.declareBoolean(Request::setForce, FORCE);
             PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH);
-            PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH_V7);
         }
 
         public static Request parseRequest(String jobId, XContentParser parser) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -62,6 +63,7 @@ private GetDatafeedsStatsAction() {
     // serialized to older nodes where the transport action was a MasterNodeReadAction.
     // TODO: Make this a simple request in a future version where there is no possibility
     // of this request being serialized to another node.
+    @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING)
     public static class Request extends MasterNodeReadRequest<Request> {
 
         public static final String ALLOW_NO_MATCH = "allow_no_match";