Merge branch 'main' into refactoring/legacy_index_version_N-3

javanna · Dec 12, 2024 · 29f2783 · 29f2783
2 parents 7ef7c3c + 80a1a6f
commit 29f2783
Show file tree

Hide file tree

Showing 91 changed files with 1,295 additions and 1,036 deletions.
diff --git a/docs/changelog/118103.yaml b/docs/changelog/118103.yaml
@@ -0,0 +1,11 @@
+pr: 118103
+summary: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7"
+area: Infra/Core
+type: breaking
+issues: []
+breaking:
+  title: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7"
+  area: REST API
+  details: "This PR removes all references to V_7 in the Rest API. V7 features marked for deprecation have been removed."
+  impact: "This change is breaking for any external plugins/clients that rely on the V_7 enum or deprecated version 7 functionality"
+  notable: false
diff --git a/docs/changelog/118375.yaml b/docs/changelog/118375.yaml
@@ -0,0 +1,5 @@
+pr: 118375
+summary: Check for presence of error object when validating streaming responses from integrations in the inference API
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/118435.yaml b/docs/changelog/118435.yaml
@@ -0,0 +1,6 @@
+pr: 118435
+summary: '`_score` should not be a reserved attribute in ES|QL'
+area: ES|QL
+type: enhancement
+issues:
+ - 118460
diff --git a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java
@@ -20,10 +20,7 @@ public enum RestApiVersion {
 
     V_9(9),
 
-    V_8(8),
-
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove all references to V_7 then delete this annotation
-    V_7(7);
+    V_8(8);
 
     public final byte major;
 
@@ -54,23 +51,18 @@ public static Predicate<RestApiVersion> equalTo(RestApiVersion restApiVersion) {
         return switch (restApiVersion) {
             case V_9 -> r -> r.major == V_9.major;
             case V_8 -> r -> r.major == V_8.major;
-            case V_7 -> r -> r.major == V_7.major;
         };
     }
 
     public static Predicate<RestApiVersion> onOrAfter(RestApiVersion restApiVersion) {
         return switch (restApiVersion) {
             case V_9 -> r -> r.major >= V_9.major;
             case V_8 -> r -> r.major >= V_8.major;
-            case V_7 -> r -> r.major >= V_7.major;
         };
     }
 
     public static RestApiVersion forMajor(int major) {
         switch (major) {
-            case 7 -> {
-                return V_7;
-            }
             case 8 -> {
                 return V_8;
             }

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -180,6 +180,12 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT"
   method: "test {scoring.*}"
   issue: https://github.com/elastic/elasticsearch/issues/117641
+- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {scoring.QstrWithFieldAndScoringSortedEval}
   issue: https://github.com/elastic/elasticsearch/issues/117751
@@ -314,9 +320,9 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/118220
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   issue: https://github.com/elastic/elasticsearch/issues/118238
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testInvalidJSON
-  issue: https://github.com/elastic/elasticsearch/issues/116521
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test011SecurityEnabledStatus
+  issue: https://github.com/elastic/elasticsearch/issues/118517
 
 # Examples:
 #

diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -177,7 +177,7 @@ private static List<IngestDocument> parseDocs(Map<String, Object> config, RestAp
             String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.INDEX.getFieldName(), "_index");
             String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.ID.getFieldName(), "_id");
             String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, dataMap, Metadata.ROUTING.getFieldName());
-            if (restApiVersion != RestApiVersion.V_8 && dataMap.containsKey(Metadata.TYPE.getFieldName())) {
+            if (dataMap.containsKey(Metadata.TYPE.getFieldName())) {
                 deprecationLogger.compatibleCritical(
                     "simulate_pipeline_with_types",
                     "[types removal] specifying _type in pipeline simulation requests is deprecated"

diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
@@ -49,7 +49,6 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) {
      */
     default Iterator<? extends ToXContent> toXContentChunked(RestApiVersion restApiVersion, ToXContent.Params params) {
         return switch (restApiVersion) {
-            case V_7 -> throw new AssertionError("v7 API not supported");
             case V_8 -> toXContentChunkedV8(params);
             case V_9 -> toXContentChunked(params);
         };

diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
@@ -957,6 +958,8 @@ void resetTerminate() {
         terminate = false;
     }
 
+    // Unconditionally deprecate the _type field once V7 BWC support is removed
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     public enum Metadata {
         INDEX(IndexFieldMapper.NAME),
         TYPE("_type"),

diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java
@@ -30,7 +30,7 @@ public class BulkRequestParserTests extends ESTestCase {
 
     @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) // Replace with just RestApiVersion.values() when V8 no longer exists
     public static final List<RestApiVersion> REST_API_VERSIONS_POST_V8 = Stream.of(RestApiVersion.values())
-        .filter(v -> v.compareTo(RestApiVersion.V_8) > 0)
+        .filter(v -> v.matches(RestApiVersion.onOrAfter(RestApiVersion.V_9)))
         .toList();
 
     public void testParserCannotBeReusedAfterFailure() {

diff --git a/...er/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/...er/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -342,7 +342,7 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception {
             requestContent,
             false,
             ingestService,
-            RestApiVersion.V_7
+            RestApiVersion.V_8
         );
         assertThat(actualRequest.verbose(), equalTo(false));
         assertThat(actualRequest.documents().size(), equalTo(numDocs));

diff --git a/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java b/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java
@@ -11,6 +11,7 @@
 
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterIndexInput;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -267,17 +268,47 @@ public void testSkipBytes() throws Exception {
             skipBytesExpected
         );
 
-        IndexInput input = dir.openInput("test", IOContext.DEFAULT);
-        InputStreamIndexInput is = new InputStreamIndexInput(input, limit);
+        var countingInput = new CountingReadBytesIndexInput("test", dir.openInput("test", IOContext.DEFAULT));
+        InputStreamIndexInput is = new InputStreamIndexInput(countingInput, limit);
         is.readNBytes(initialReadBytes);
         assertThat(is.skip(skipBytes), equalTo((long) skipBytesExpected));
+        long expectedActualInitialBytesRead = Math.min(Math.min(initialReadBytes, limit), bytes);
+        assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead));
 
         int remainingBytes = Math.min(bytes, limit) - seekExpected;
         for (int i = seekExpected; i < seekExpected + remainingBytes; i++) {
             assertThat(is.read(), equalTo(i));
         }
+        assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead + remainingBytes));
     }
 
+    protected static class CountingReadBytesIndexInput extends FilterIndexInput {
+        private long bytesRead = 0;
+
+        public CountingReadBytesIndexInput(String resourceDescription, IndexInput in) {
+            super(resourceDescription, in);
+        }
+
+        @Override
+        public byte readByte() throws IOException {
+            long filePointerBefore = getFilePointer();
+            byte b = super.readByte();
+            bytesRead += getFilePointer() - filePointerBefore;
+            return b;
+        }
+
+        @Override
+        public void readBytes(byte[] b, int offset, int len) throws IOException {
+            long filePointerBefore = getFilePointer();
+            super.readBytes(b, offset, len);
+            bytesRead += getFilePointer() - filePointerBefore;
+        }
+
+        public long getBytesRead() {
+            return bytesRead;
+        }
+    };
+
     public void testReadZeroShouldReturnZero() throws IOException {
         try (Directory dir = new ByteBuffersDirectory()) {
             try (IndexOutput output = dir.createOutput("test", IOContext.DEFAULT)) {

diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.junit.After;
 import org.junit.Before;
+import org.mockito.Mockito;
 import org.mockito.stubbing.Answer;
 
 import java.io.IOException;
@@ -315,7 +316,13 @@ public void testInvalidJSON() throws Exception {
         writeTestFile(fileSettingsService.watchedFile(), "test_invalid_JSON");
         awaitOrBust(fileChangeBarrier);
 
-        verify(fileSettingsService, times(1)).onProcessFileChangesException(
+        // These checks use atLeast(1) because the initial JSON is also invalid,
+        // and so we sometimes get two calls to these error-reporting methods
+        // depending on timing. Rather than trace down the root cause and fix
+        // it, we tolerate this for now because, hey, invalid JSON is invalid JSON
+        // and this is still testing what we want to test.
+
+        verify(fileSettingsService, Mockito.atLeast(1)).onProcessFileChangesException(
             argThat(e -> unwrapException(e) instanceof XContentParseException)
         );
 
@@ -324,7 +331,7 @@ public void testInvalidJSON() throws Exception {
         // of the watcher thread itself, which occurs asynchronously when clusterChanged is first called.
 
         assertEquals(YELLOW, healthIndicatorService.calculate(false, null).status());
-        verify(healthIndicatorService).failureOccurred(contains(XContentParseException.class.getName()));
+        verify(healthIndicatorService, Mockito.atLeast(1)).failureOccurred(contains(XContentParseException.class.getName()));
     }
 
     /**

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -142,7 +141,7 @@ static boolean isEnterprise(String typeName) {
      */
     public static final String LICENSE_VERSION_MODE = "license_version";
     /**
-     * Set for {@link RestApiVersion#V_7} requests only
+     * Set for RestApiVersion#V_7 requests only
      * XContent param name to map the "enterprise" license type to "platinum"
      * for backwards compatibility with older clients
      */

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitStep.java
@@ -33,22 +33,6 @@ public boolean isCompletable() {
         return true;
     }
 
-    public static class Result {
-        private final boolean complete;
-        private final ToXContentObject informationContext;
-
-        public Result(boolean complete, ToXContentObject informationContext) {
-            this.complete = complete;
-            this.informationContext = informationContext;
-        }
-
-        public boolean isComplete() {
-            return complete;
-        }
-
-        public ToXContentObject getInformationContext() {
-            return informationContext;
-        }
-    }
+    public record Result(boolean complete, ToXContentObject informationContext) {}
 
 }
diff --git a/...re/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStep.java b/...re/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStep.java
@@ -62,7 +62,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
 
         Result stepResult = stepToExecute.isConditionMet(index, clusterState);
 
-        if (stepResult.isComplete() == false) {
+        if (stepResult.complete() == false) {
             // checking the threshold after we execute the step to make sure we execute the wrapped step at least once (because time is a
             // wonderful thing)
             TimeValue retryThreshold = LifecycleSettings.LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING.get(idxMeta.getSettings());
@@ -77,7 +77,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
                     getKey().name(),
                     getKey().action(),
                     idxMeta.getIndex().getName(),
-                    Strings.toString(stepResult.getInformationContext()),
+                    Strings.toString(stepResult.informationContext()),
                     nextKeyOnThresholdBreach
                 );
                 logger.debug(message);

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java
@@ -114,29 +114,20 @@ public boolean equals(Object obj) {
         return super.equals(obj) && Objects.equals(maxNumSegments, other.maxNumSegments);
     }
 
-    public static class Info implements ToXContentObject {
-
-        private final long numberShardsLeftToMerge;
+    public record Info(long numberShardsLeftToMerge) implements ToXContentObject {
 
         static final ParseField SHARDS_TO_MERGE = new ParseField("shards_left_to_merge");
         static final ParseField MESSAGE = new ParseField("message");
         static final ConstructingObjectParser<Info, Void> PARSER = new ConstructingObjectParser<>(
             "segment_count_step_info",
             a -> new Info((long) a[0])
         );
+
         static {
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), SHARDS_TO_MERGE);
             PARSER.declareString((i, s) -> {}, MESSAGE);
         }
 
-        public Info(long numberShardsLeftToMerge) {
-            this.numberShardsLeftToMerge = numberShardsLeftToMerge;
-        }
-
-        public long getNumberShardsLeftToMerge() {
-            return numberShardsLeftToMerge;
-        }
-
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
@@ -150,23 +141,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder;
         }
 
-        @Override
-        public int hashCode() {
-            return Objects.hash(numberShardsLeftToMerge);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (obj == null) {
-                return false;
-            }
-            if (getClass() != obj.getClass()) {
-                return false;
-            }
-            Info other = (Info) obj;
-            return Objects.equals(numberShardsLeftToMerge, other.numberShardsLeftToMerge);
-        }
-
         @Override
         public String toString() {
             return Strings.toString(this);