Merge branch 'main' into refactoring/legacy_index_version_N-3

docs/changelog/118103.yaml

@@ -0,0 +1,11 @@
+pr: 118103
+summary: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7"
+area: Infra/Core
+type: breaking
+issues: []
+breaking:
+  title: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7"
+  area: REST API
+  details: "This PR removes all references to V_7 in the Rest API. V7 features marked for deprecation have been removed."
+  impact: "This change is breaking for any external plugins/clients that rely on the V_7 enum or deprecated version 7 functionality"
+  notable: false

docs/changelog/118375.yaml

@@ -0,0 +1,5 @@
+pr: 118375
+summary: Check for presence of error object when validating streaming responses from integrations in the inference API
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/118435.yaml

@@ -0,0 +1,6 @@
+pr: 118435
+summary: '`_score` should not be a reserved attribute in ES|QL'
+area: ES|QL
+type: enhancement
+issues:
+ - 118460

libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java

@@ -20,10 +20,7 @@ public enum RestApiVersion {
 
     V_9(9),
 
-    V_8(8),
-
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove all references to V_7 then delete this annotation
-    V_7(7);
+    V_8(8);
 
     public final byte major;
 
@@ -54,23 +51,18 @@ public static Predicate<RestApiVersion> equalTo(RestApiVersion restApiVersion) {
         return switch (restApiVersion) {
             case V_9 -> r -> r.major == V_9.major;
             case V_8 -> r -> r.major == V_8.major;
-            case V_7 -> r -> r.major == V_7.major;
         };
     }
 
     public static Predicate<RestApiVersion> onOrAfter(RestApiVersion restApiVersion) {
         return switch (restApiVersion) {
             case V_9 -> r -> r.major >= V_9.major;
             case V_8 -> r -> r.major >= V_8.major;
-            case V_7 -> r -> r.major >= V_7.major;
         };
     }
 
     public static RestApiVersion forMajor(int major) {
         switch (major) {
-            case 7 -> {
-                return V_7;
-            }
             case 8 -> {
                 return V_8;
             }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -18,7 +18,6 @@
 import java.lang.module.ModuleFinder;
 import java.lang.module.ModuleReference;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
@@ -56,8 +55,8 @@ public <E extends Entitlement> Stream<E> getEntitlements(Class<E> entitlementCla
 
     final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new HashMap<>();
 
-    protected final Policy serverPolicy;
-    protected final Map<String, Policy> pluginPolicies;
+    protected final Map<String, List<Entitlement>> serverEntitlements;
+    protected final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, String> pluginResolver;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
@@ -79,19 +78,16 @@ private static Set<Module> findSystemModules() {
     }
 
     public PolicyManager(Policy defaultPolicy, Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
-        this.serverPolicy = Objects.requireNonNull(defaultPolicy);
-        this.pluginPolicies = Collections.unmodifiableMap(Objects.requireNonNull(pluginPolicies));
+        this.serverEntitlements = buildScopeEntitlementsMap(Objects.requireNonNull(defaultPolicy));
+        this.pluginsEntitlements = Objects.requireNonNull(pluginPolicies)
+            .entrySet()
+            .stream()
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue())));
         this.pluginResolver = pluginResolver;
     }
 
-    private static List<Entitlement> lookupEntitlementsForModule(Policy policy, String moduleName) {
-        for (int i = 0; i < policy.scopes.size(); ++i) {
-            var scope = policy.scopes.get(i);
-            if (scope.name.equals(moduleName)) {
-                return scope.entitlements;
-            }
-        }
-        return null;
+    private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy policy) {
+        return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements));
     }
 
     public void checkExitVM(Class<?> callerClass) {
@@ -141,21 +137,21 @@ ModuleEntitlements getEntitlementsOrThrow(Class<?> callerClass, Module requestin
 
         if (isServerModule(requestingModule)) {
             var scopeName = requestingModule.getName();
-            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverPolicy, scopeName);
+            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName);
         }
 
         // plugins
         var pluginName = pluginResolver.apply(callerClass);
         if (pluginName != null) {
-            var pluginPolicy = pluginPolicies.get(pluginName);
-            if (pluginPolicy != null) {
+            var pluginEntitlements = pluginsEntitlements.get(pluginName);
+            if (pluginEntitlements != null) {
                 final String scopeName;
                 if (requestingModule.isNamed() == false) {
                     scopeName = ALL_UNNAMED;
                 } else {
                     scopeName = requestingModule.getName();
                 }
-                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginPolicy, scopeName);
+                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName);
             }
         }
 
@@ -167,15 +163,20 @@ private static String buildModuleNoPolicyMessage(Class<?> callerClass, Module re
         return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName());
     }
 
-    private ModuleEntitlements getModuleEntitlementsOrThrow(Class<?> callerClass, Module module, Policy policy, String moduleName) {
-        var entitlements = lookupEntitlementsForModule(policy, moduleName);
+    private ModuleEntitlements getModuleEntitlementsOrThrow(
+        Class<?> callerClass,
+        Module module,
+        Map<String, List<Entitlement>> scopeEntitlements,
+        String moduleName
+    ) {
+        var entitlements = scopeEntitlements.get(moduleName);
         if (entitlements == null) {
             // Module without entitlements - remember we don't have any
             moduleEntitlementsMap.put(module, ModuleEntitlements.NONE);
             throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module));
         }
         // We have a policy for this module
-        var classEntitlements = createClassEntitlements(entitlements);
+        var classEntitlements = new ModuleEntitlements(entitlements);
         moduleEntitlementsMap.put(module, classEntitlements);
         return classEntitlements;
     }
@@ -184,10 +185,6 @@ private static boolean isServerModule(Module requestingModule) {
         return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot();
     }
 
-    private ModuleEntitlements createClassEntitlements(List<Entitlement> entitlements) {
-        return new ModuleEntitlements(entitlements);
-    }
-
     private static Module requestingModule(Class<?> callerClass) {
         if (callerClass != null) {
             Module callerModule = callerClass.getModule();
@@ -222,6 +219,6 @@ private static boolean isTriviallyAllowed(Module requestingModule) {
 
     @Override
     public String toString() {
-        return "PolicyManager{" + "serverPolicy=" + serverPolicy + ", pluginPolicies=" + pluginPolicies + '}';
+        return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}';
     }
 }

muted-tests.yml

@@ -180,6 +180,12 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT"
   method: "test {scoring.*}"
   issue: https://github.com/elastic/elasticsearch/issues/117641
+- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {scoring.QstrWithFieldAndScoringSortedEval}
   issue: https://github.com/elastic/elasticsearch/issues/117751
@@ -314,9 +320,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/118220
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   issue: https://github.com/elastic/elasticsearch/issues/118238
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testInvalidJSON
-  issue: https://github.com/elastic/elasticsearch/issues/116521
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test011SecurityEnabledStatus
+  issue: https://github.com/elastic/elasticsearch/issues/118517
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

@@ -177,7 +177,7 @@ private static List<IngestDocument> parseDocs(Map<String, Object> config, RestAp
             String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.INDEX.getFieldName(), "_index");
             String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.ID.getFieldName(), "_id");
             String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, dataMap, Metadata.ROUTING.getFieldName());
-            if (restApiVersion != RestApiVersion.V_8 && dataMap.containsKey(Metadata.TYPE.getFieldName())) {
+            if (dataMap.containsKey(Metadata.TYPE.getFieldName())) {
                 deprecationLogger.compatibleCritical(
                     "simulate_pipeline_with_types",
                     "[types removal] specifying _type in pipeline simulation requests is deprecated"

server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java

@@ -49,7 +49,6 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) {
      */
     default Iterator<? extends ToXContent> toXContentChunked(RestApiVersion restApiVersion, ToXContent.Params params) {
         return switch (restApiVersion) {
-            case V_7 -> throw new AssertionError("v7 API not supported");
             case V_8 -> toXContentChunkedV8(params);
             case V_9 -> toXContentChunked(params);
         };

server/src/main/java/org/elasticsearch/ingest/IngestDocument.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
@@ -957,6 +958,8 @@ void resetTerminate() {
         terminate = false;
     }
 
+    // Unconditionally deprecate the _type field once V7 BWC support is removed
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     public enum Metadata {
         INDEX(IndexFieldMapper.NAME),
         TYPE("_type"),

server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java

@@ -416,7 +416,7 @@ static String toModuleName(String name) {
         return result;
     }
 
-    static final String toPackageName(String className) {
+    static String toPackageName(String className) {
         assert className.endsWith(".") == false;
         int index = className.lastIndexOf('.');
         if (index == -1) {
@@ -426,11 +426,11 @@ static final String toPackageName(String className) {
     }
 
     @SuppressForbidden(reason = "I need to convert URL's to Paths")
-    static final Path[] urlsToPaths(Set<URL> urls) {
+    static Path[] urlsToPaths(Set<URL> urls) {
         return urls.stream().map(PluginsLoader::uncheckedToURI).map(PathUtils::get).toArray(Path[]::new);
     }
 
-    static final URI uncheckedToURI(URL url) {
+    static URI uncheckedToURI(URL url) {
         try {
             return url.toURI();
         } catch (URISyntaxException e) {

server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java

@@ -30,7 +30,7 @@ public class BulkRequestParserTests extends ESTestCase {
 
     @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) // Replace with just RestApiVersion.values() when V8 no longer exists
     public static final List<RestApiVersion> REST_API_VERSIONS_POST_V8 = Stream.of(RestApiVersion.values())
-        .filter(v -> v.compareTo(RestApiVersion.V_8) > 0)
+        .filter(v -> v.matches(RestApiVersion.onOrAfter(RestApiVersion.V_9)))
         .toList();
 
     public void testParserCannotBeReusedAfterFailure() {

server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java

@@ -342,7 +342,7 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception {
             requestContent,
             false,
             ingestService,
-            RestApiVersion.V_7
+            RestApiVersion.V_8
         );
         assertThat(actualRequest.verbose(), equalTo(false));
         assertThat(actualRequest.documents().size(), equalTo(numDocs));

server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java

@@ -11,6 +11,7 @@
 
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterIndexInput;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -267,17 +268,47 @@ public void testSkipBytes() throws Exception {
             skipBytesExpected
         );
 
-        IndexInput input = dir.openInput("test", IOContext.DEFAULT);
-        InputStreamIndexInput is = new InputStreamIndexInput(input, limit);
+        var countingInput = new CountingReadBytesIndexInput("test", dir.openInput("test", IOContext.DEFAULT));
+        InputStreamIndexInput is = new InputStreamIndexInput(countingInput, limit);
         is.readNBytes(initialReadBytes);
         assertThat(is.skip(skipBytes), equalTo((long) skipBytesExpected));
+        long expectedActualInitialBytesRead = Math.min(Math.min(initialReadBytes, limit), bytes);
+        assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead));
 
         int remainingBytes = Math.min(bytes, limit) - seekExpected;
         for (int i = seekExpected; i < seekExpected + remainingBytes; i++) {
             assertThat(is.read(), equalTo(i));
         }
+        assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead + remainingBytes));
     }
 
+    protected static class CountingReadBytesIndexInput extends FilterIndexInput {
+        private long bytesRead = 0;
+
+        public CountingReadBytesIndexInput(String resourceDescription, IndexInput in) {
+            super(resourceDescription, in);
+        }
+
+        @Override
+        public byte readByte() throws IOException {
+            long filePointerBefore = getFilePointer();
+            byte b = super.readByte();
+            bytesRead += getFilePointer() - filePointerBefore;
+            return b;
+        }
+
+        @Override
+        public void readBytes(byte[] b, int offset, int len) throws IOException {
+            long filePointerBefore = getFilePointer();
+            super.readBytes(b, offset, len);
+            bytesRead += getFilePointer() - filePointerBefore;
+        }
+
+        public long getBytesRead() {
+            return bytesRead;
+        }
+    };
+
     public void testReadZeroShouldReturnZero() throws IOException {
         try (Directory dir = new ByteBuffersDirectory()) {
             try (IndexOutput output = dir.createOutput("test", IOContext.DEFAULT)) {