From 9056fe430d0679593ce140a27c658215ba84415a Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 11 Dec 2024 15:54:03 -0500 Subject: [PATCH 01/24] Convert some ILM classes to records (#118466) --- .../xpack/core/ilm/ClusterStateWaitStep.java | 18 +-- .../ClusterStateWaitUntilThresholdStep.java | 4 +- .../xpack/core/ilm/SegmentCountStep.java | 30 +---- .../core/ilm/WaitForFollowShardTasksStep.java | 106 ++++-------------- .../core/ilm/step/info/AllocationInfo.java | 53 +-------- .../ilm/step/info/SingleMessageFieldInfo.java | 31 +---- .../core/ilm/AllocationRoutedStepTests.java | 12 +- .../CheckNoDataStreamWriteIndexStepTests.java | 14 +-- .../core/ilm/CheckShrinkReadyStepTests.java | 14 +-- .../ilm/CheckTargetShardsCountStepTests.java | 8 +- ...usterStateWaitUntilThresholdStepTests.java | 30 ++--- .../ilm/DataTierMigrationRoutedStepTests.java | 32 +++--- .../core/ilm/SegmentCountStepInfoTests.java | 2 +- .../ilm/ShrunkShardsAllocatedStepTests.java | 12 +- .../core/ilm/ShrunkenIndexCheckStepTests.java | 12 +- .../core/ilm/WaitForActiveShardsTests.java | 16 +-- .../core/ilm/WaitForDataTierStepTests.java | 6 +- .../WaitForFollowShardTasksStepInfoTests.java | 2 +- .../ilm/WaitForFollowShardTasksStepTests.java | 8 +- .../core/ilm/WaitForIndexColorStepTests.java | 48 ++++---- .../ilm/WaitForIndexingCompleteStepTests.java | 18 +-- .../info/AllocationRoutedStepInfoTests.java | 12 +- ...adataMigrateToDataTiersRoutingService.java | 62 ++-------- .../xpack/ilm/ExecuteStepsUpdateTask.java | 4 +- .../TransportMigrateToDataTiersAction.java | 24 ++-- ...MigrateToDataTiersRoutingServiceTests.java | 42 +++---- 26 files changed, 199 insertions(+), 421 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitStep.java index d1dbfede63c60..4ed83fa170ead 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitStep.java @@ -33,22 +33,6 @@ public boolean isCompletable() { return true; } - public static class Result { - private final boolean complete; - private final ToXContentObject informationContext; - - public Result(boolean complete, ToXContentObject informationContext) { - this.complete = complete; - this.informationContext = informationContext; - } - - public boolean isComplete() { - return complete; - } - - public ToXContentObject getInformationContext() { - return informationContext; - } - } + public record Result(boolean complete, ToXContentObject informationContext) {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStep.java index 5e30baa6b9669..c7fa1ea611a0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStep.java @@ -62,7 +62,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) { Result stepResult = stepToExecute.isConditionMet(index, clusterState); - if (stepResult.isComplete() == false) { + if (stepResult.complete() == false) { // checking the threshold after we execute the step to make sure we execute the wrapped step at least once (because time is a // wonderful thing) TimeValue retryThreshold = LifecycleSettings.LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING.get(idxMeta.getSettings()); @@ -77,7 +77,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) { getKey().name(), getKey().action(), idxMeta.getIndex().getName(), - Strings.toString(stepResult.getInformationContext()), + Strings.toString(stepResult.informationContext()), nextKeyOnThresholdBreach ); logger.debug(message); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java index 800ea603ede8c..ad8f450fb0849 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SegmentCountStep.java @@ -114,9 +114,7 @@ public boolean equals(Object obj) { return super.equals(obj) && Objects.equals(maxNumSegments, other.maxNumSegments); } - public static class Info implements ToXContentObject { - - private final long numberShardsLeftToMerge; + public record Info(long numberShardsLeftToMerge) implements ToXContentObject { static final ParseField SHARDS_TO_MERGE = new ParseField("shards_left_to_merge"); static final ParseField MESSAGE = new ParseField("message"); @@ -124,19 +122,12 @@ public static class Info implements ToXContentObject { "segment_count_step_info", a -> new Info((long) a[0]) ); + static { PARSER.declareLong(ConstructingObjectParser.constructorArg(), SHARDS_TO_MERGE); PARSER.declareString((i, s) -> {}, MESSAGE); } - public Info(long numberShardsLeftToMerge) { - this.numberShardsLeftToMerge = numberShardsLeftToMerge; - } - - public long getNumberShardsLeftToMerge() { - return numberShardsLeftToMerge; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -150,23 +141,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public int hashCode() { - return Objects.hash(numberShardsLeftToMerge); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Info other = (Info) obj; - return Objects.equals(numberShardsLeftToMerge, other.numberShardsLeftToMerge); - } - @Override public String toString() { return Strings.toString(this); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java index f1fbdde1e9a5d..224319722297c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStep.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ilm.UnfollowAction.CCR_METADATA_KEY; @@ -70,9 +69,9 @@ static void handleResponse(FollowStatsAction.StatsResponses responses, Listener if (conditionMet) { listener.onResponse(true, null); } else { - List shardFollowTaskInfos = unSyncedShardFollowStatuses.stream() + List shardFollowTaskInfos = unSyncedShardFollowStatuses.stream() .map( - status -> new Info.ShardFollowTaskInfo( + status -> new ShardFollowTaskInfo( status.followerIndex(), status.getShardId(), status.leaderGlobalCheckpoint(), @@ -84,21 +83,11 @@ static void handleResponse(FollowStatsAction.StatsResponses responses, Listener } } - static final class Info implements ToXContentObject { + record Info(List shardFollowTaskInfos) implements ToXContentObject { static final ParseField SHARD_FOLLOW_TASKS = new ParseField("shard_follow_tasks"); static final ParseField MESSAGE = new ParseField("message"); - private final List shardFollowTaskInfos; - - Info(List shardFollowTaskInfos) { - this.shardFollowTaskInfos = shardFollowTaskInfos; - } - - List getShardFollowTaskInfos() { - return shardFollowTaskInfos; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -114,85 +103,30 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Info info = (Info) o; - return Objects.equals(shardFollowTaskInfos, info.shardFollowTaskInfos); - } - - @Override - public int hashCode() { - return Objects.hash(shardFollowTaskInfos); - } - @Override public String toString() { return Strings.toString(this); } + } - static final class ShardFollowTaskInfo implements ToXContentObject { - - static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index"); - static final ParseField SHARD_ID_FIELD = new ParseField("shard_id"); - static final ParseField LEADER_GLOBAL_CHECKPOINT_FIELD = new ParseField("leader_global_checkpoint"); - static final ParseField FOLLOWER_GLOBAL_CHECKPOINT_FIELD = new ParseField("follower_global_checkpoint"); - - private final String followerIndex; - private final int shardId; - private final long leaderGlobalCheckpoint; - private final long followerGlobalCheckpoint; - - ShardFollowTaskInfo(String followerIndex, int shardId, long leaderGlobalCheckpoint, long followerGlobalCheckpoint) { - this.followerIndex = followerIndex; - this.shardId = shardId; - this.leaderGlobalCheckpoint = leaderGlobalCheckpoint; - this.followerGlobalCheckpoint = followerGlobalCheckpoint; - } - - String getFollowerIndex() { - return followerIndex; - } - - int getShardId() { - return shardId; - } - - long getLeaderGlobalCheckpoint() { - return leaderGlobalCheckpoint; - } - - long getFollowerGlobalCheckpoint() { - return followerGlobalCheckpoint; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex); - builder.field(SHARD_ID_FIELD.getPreferredName(), shardId); - builder.field(LEADER_GLOBAL_CHECKPOINT_FIELD.getPreferredName(), leaderGlobalCheckpoint); - builder.field(FOLLOWER_GLOBAL_CHECKPOINT_FIELD.getPreferredName(), followerGlobalCheckpoint); - builder.endObject(); - return builder; - } + record ShardFollowTaskInfo(String followerIndex, int shardId, long leaderGlobalCheckpoint, long followerGlobalCheckpoint) + implements + ToXContentObject { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ShardFollowTaskInfo that = (ShardFollowTaskInfo) o; - return shardId == that.shardId - && leaderGlobalCheckpoint == that.leaderGlobalCheckpoint - && followerGlobalCheckpoint == that.followerGlobalCheckpoint - && Objects.equals(followerIndex, that.followerIndex); - } + static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index"); + static final ParseField SHARD_ID_FIELD = new ParseField("shard_id"); + static final ParseField LEADER_GLOBAL_CHECKPOINT_FIELD = new ParseField("leader_global_checkpoint"); + static final ParseField FOLLOWER_GLOBAL_CHECKPOINT_FIELD = new ParseField("follower_global_checkpoint"); - @Override - public int hashCode() { - return Objects.hash(followerIndex, shardId, leaderGlobalCheckpoint, followerGlobalCheckpoint); - } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex); + builder.field(SHARD_ID_FIELD.getPreferredName(), shardId); + builder.field(LEADER_GLOBAL_CHECKPOINT_FIELD.getPreferredName(), leaderGlobalCheckpoint); + builder.field(FOLLOWER_GLOBAL_CHECKPOINT_FIELD.getPreferredName(), followerGlobalCheckpoint); + builder.endObject(); + return builder; } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationInfo.java index 5732f5e72a42f..9f280bd344083 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationInfo.java @@ -14,19 +14,15 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Objects; /** * Represents the state of an index's shards allocation, including a user friendly message describing the current state. * It allows to transfer the allocation information to {@link org.elasticsearch.xcontent.XContent} using * {@link #toXContent(XContentBuilder, Params)} */ -public class AllocationInfo implements ToXContentObject { - - private final long numberOfReplicas; - private final long numberShardsLeftToAllocate; - private final boolean allShardsActive; - private final String message; +public record AllocationInfo(long numberOfReplicas, long numberShardsLeftToAllocate, boolean allShardsActive, String message) + implements + ToXContentObject { static final ParseField NUMBER_OF_REPLICAS = new ParseField("number_of_replicas"); static final ParseField SHARDS_TO_ALLOCATE = new ParseField("shards_left_to_allocate"); @@ -44,13 +40,6 @@ public class AllocationInfo implements ToXContentObject { PARSER.declareString(ConstructingObjectParser.constructorArg(), MESSAGE); } - public AllocationInfo(long numberOfReplicas, long numberShardsLeftToAllocate, boolean allShardsActive, String message) { - this.numberOfReplicas = numberOfReplicas; - this.numberShardsLeftToAllocate = numberShardsLeftToAllocate; - this.allShardsActive = allShardsActive; - this.message = message; - } - /** * Builds the AllocationInfo representing a cluster state with a routing table that does not have enough shards active for a * particular index. @@ -72,22 +61,6 @@ public static AllocationInfo allShardsActiveAllocationInfo(long numReplicas, lon ); } - public long getNumberOfReplicas() { - return numberOfReplicas; - } - - public long getNumberShardsLeftToAllocate() { - return numberShardsLeftToAllocate; - } - - public boolean allShardsActive() { - return allShardsActive; - } - - public String getMessage() { - return message; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -99,26 +72,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public int hashCode() { - return Objects.hash(numberOfReplicas, numberShardsLeftToAllocate, allShardsActive); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - AllocationInfo other = (AllocationInfo) obj; - return Objects.equals(numberOfReplicas, other.numberOfReplicas) - && Objects.equals(numberShardsLeftToAllocate, other.numberShardsLeftToAllocate) - && Objects.equals(message, other.message) - && Objects.equals(allShardsActive, other.allShardsActive); - } - @Override public String toString() { return Strings.toString(this); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/SingleMessageFieldInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/SingleMessageFieldInfo.java index 8d7eb8c3d303b..bd23e21d46489 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/SingleMessageFieldInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/step/info/SingleMessageFieldInfo.java @@ -12,20 +12,13 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Objects; /** * A simple object that allows a `message` field to be transferred to `XContent`. */ -public class SingleMessageFieldInfo implements ToXContentObject { +public record SingleMessageFieldInfo(String message) implements ToXContentObject { - private final String message; - - static final ParseField MESSAGE = new ParseField("message"); - - public SingleMessageFieldInfo(String message) { - this.message = message; - } + private static final ParseField MESSAGE = new ParseField("message"); @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { @@ -35,24 +28,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public String getMessage() { - return message; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SingleMessageFieldInfo that = (SingleMessageFieldInfo) o; - return Objects.equals(message, that.message); - } - - @Override - public int hashCode() { - return Objects.hash(message); - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AllocationRoutedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AllocationRoutedStepTests.java index 415014623f340..afad708ddbe2c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AllocationRoutedStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/AllocationRoutedStepTests.java @@ -181,8 +181,8 @@ public void testClusterExcludeFiltersConditionMetOnlyOneCopyAllocated() { Result actualResult = step.isConditionMet(index, clusterState); Result expectedResult = new ClusterStateWaitStep.Result(false, allShardsActiveAllocationInfo(1, 1)); - assertEquals(expectedResult.isComplete(), actualResult.isComplete()); - assertEquals(expectedResult.getInformationContext(), actualResult.getInformationContext()); + assertEquals(expectedResult.complete(), actualResult.complete()); + assertEquals(expectedResult.informationContext(), actualResult.informationContext()); } public void testExcludeConditionMetOnlyOneCopyAllocated() { @@ -495,8 +495,8 @@ public void testExecuteIndexMissing() throws Exception { AllocationRoutedStep step = createRandomInstance(); Result actualResult = step.isConditionMet(index, clusterState); - assertFalse(actualResult.isComplete()); - assertNull(actualResult.getInformationContext()); + assertFalse(actualResult.complete()); + assertNull(actualResult.informationContext()); } private void assertAllocateStatus( @@ -537,7 +537,7 @@ private void assertAllocateStatus( .routingTable(RoutingTable.builder().add(indexRoutingTable).build()) .build(); Result actualResult = step.isConditionMet(index, clusterState); - assertEquals(expectedResult.isComplete(), actualResult.isComplete()); - assertEquals(expectedResult.getInformationContext(), actualResult.getInformationContext()); + assertEquals(expectedResult.complete(), actualResult.complete()); + assertEquals(expectedResult.informationContext(), actualResult.informationContext()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java index af9aa0982d61d..54c6ceb814af8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java @@ -59,8 +59,8 @@ public void testStepCompleteIfIndexIsNotPartOfDataStream() { .build(); ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), is(nullValue())); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), is(nullValue())); } public void testStepIncompleteIfIndexIsTheDataStreamWriteIndex() { @@ -94,10 +94,10 @@ public void testStepIncompleteIfIndexIsTheDataStreamWriteIndex() { IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata; String expectedIndexName = indexToOperateOn.getIndex().getName(); ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexToOperateOn.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat( - info.getMessage(), + info.message(), is( "index [" + expectedIndexName @@ -161,7 +161,7 @@ public void testStepCompleteIfPartOfDataStreamButNotWriteIndex() { boolean useFailureStore = randomBoolean(); IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata; ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexToOperateOn.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), is(nullValue())); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), is(nullValue())); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java index 371f7def67c52..8dcd8fc7ddd55 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckShrinkReadyStepTests.java @@ -417,8 +417,8 @@ public void testExecuteIndexMissing() throws Exception { CheckShrinkReadyStep step = createRandomInstance(); ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState); - assertFalse(actualResult.isComplete()); - assertNull(actualResult.getInformationContext()); + assertFalse(actualResult.complete()); + assertNull(actualResult.informationContext()); } public void testStepCompletableIfAllShardsActive() { @@ -495,7 +495,7 @@ public void testStepCompletableIfAllShardsActive() { .build(); assertTrue(step.isCompletable()); ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState); - assertTrue(actualResult.isComplete()); + assertTrue(actualResult.complete()); assertTrue(step.isCompletable()); } } @@ -574,9 +574,9 @@ public void testStepBecomesUncompletable() { .build(); assertTrue(step.isCompletable()); ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState); - assertFalse(actualResult.isComplete()); + assertFalse(actualResult.complete()); assertThat( - Strings.toString(actualResult.getInformationContext()), + Strings.toString(actualResult.informationContext()), containsString("node with id [node1] is currently marked as shutting down") ); assertFalse(step.isCompletable()); @@ -625,8 +625,8 @@ private void assertAllocateStatus( .routingTable(RoutingTable.builder().add(indexRoutingTable).build()) .build(); ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState); - assertEquals(expectedResult.isComplete(), actualResult.isComplete()); - assertEquals(expectedResult.getInformationContext(), actualResult.getInformationContext()); + assertEquals(expectedResult.complete(), actualResult.complete()); + assertEquals(expectedResult.informationContext(), actualResult.informationContext()); } public static UnassignedInfo randomUnassignedInfo(String message) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckTargetShardsCountStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckTargetShardsCountStepTests.java index 8eb8d0f395aba..23d24fbd28730 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckTargetShardsCountStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckTargetShardsCountStepTests.java @@ -56,7 +56,7 @@ public void testStepCompleteIfTargetShardsCountIsValid() { CheckTargetShardsCountStep checkTargetShardsCountStep = new CheckTargetShardsCountStep(randomStepKey(), randomStepKey(), 2); ClusterStateWaitStep.Result result = checkTargetShardsCountStep.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); + assertThat(result.complete(), is(true)); } public void testStepIncompleteIfTargetShardsCountNotValid() { @@ -75,10 +75,10 @@ public void testStepIncompleteIfTargetShardsCountNotValid() { CheckTargetShardsCountStep checkTargetShardsCountStep = new CheckTargetShardsCountStep(randomStepKey(), randomStepKey(), 3); ClusterStateWaitStep.Result result = checkTargetShardsCountStep.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat( - info.getMessage(), + info.message(), is( "lifecycle action of policy [" + policyName diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStepTests.java index ea583b51c4c28..f24f0f86de7db 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ClusterStateWaitUntilThresholdStepTests.java @@ -70,8 +70,8 @@ public void testIndexIsMissingReturnsIncompleteResult() { new Index("testName", UUID.randomUUID().toString()), ClusterState.EMPTY_STATE ); - assertThat(result.isComplete(), is(false)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(false)); + assertThat(result.informationContext(), nullValue()); } public void testIsConditionMetForUnderlyingStep() { @@ -95,8 +95,8 @@ public void testIsConditionMetForUnderlyingStep() { ClusterStateWaitUntilThresholdStep underTest = new ClusterStateWaitUntilThresholdStep(stepToExecute, randomStepKey()); ClusterStateWaitStep.Result result = underTest.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); } { @@ -120,10 +120,10 @@ public void testIsConditionMetForUnderlyingStep() { ClusterStateWaitUntilThresholdStep underTest = new ClusterStateWaitUntilThresholdStep(stepToExecute, randomStepKey()); ClusterStateWaitStep.Result result = underTest.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - assertThat(result.getInformationContext(), notNullValue()); + assertThat(result.complete(), is(false)); + assertThat(result.informationContext(), notNullValue()); WaitForIndexingCompleteStep.IndexingNotCompleteInfo info = (WaitForIndexingCompleteStep.IndexingNotCompleteInfo) result - .getInformationContext(); + .informationContext(); assertThat( info.getMessage(), equalTo( @@ -154,8 +154,8 @@ public void testIsConditionMetForUnderlyingStep() { ClusterStateWaitUntilThresholdStep underTest = new ClusterStateWaitUntilThresholdStep(stepToExecute, nextKeyOnThresholdBreach); ClusterStateWaitStep.Result result = underTest.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); assertThat(underTest.getNextStepKey(), is(not(nextKeyOnThresholdBreach))); assertThat(underTest.getNextStepKey(), is(stepToExecute.getNextStepKey())); } @@ -184,11 +184,11 @@ public void testIsConditionMetForUnderlyingStep() { ClusterStateWaitUntilThresholdStep underTest = new ClusterStateWaitUntilThresholdStep(stepToExecute, nextKeyOnThresholdBreach); ClusterStateWaitStep.Result result = underTest.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), notNullValue()); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), notNullValue()); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat( - info.getMessage(), + info.message(), equalTo( "[" + currentStepKey.name() @@ -267,7 +267,7 @@ public boolean isRetryable() { new StepKey("phase", "action", "breached") ); - assertFalse(step.isConditionMet(indexMetadata.getIndex(), clusterState).isComplete()); + assertFalse(step.isConditionMet(indexMetadata.getIndex(), clusterState).complete()); assertThat(step.getNextStepKey().name(), equalTo("next-key")); @@ -290,7 +290,7 @@ public boolean isRetryable() { }, new StepKey("phase", "action", "breached") ); - assertTrue(step.isConditionMet(indexMetadata.getIndex(), clusterState).isComplete()); + assertTrue(step.isConditionMet(indexMetadata.getIndex(), clusterState).complete()); assertThat(step.getNextStepKey().name(), equalTo("breached")); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java index 95c1f5c4aa96b..51d1464ed5525 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java @@ -89,8 +89,8 @@ public void testExecuteWithUnassignedShard() { Result expectedResult = new Result(false, waitingForActiveShardsAllocationInfo(1)); Result actualResult = step.isConditionMet(index, clusterState); - assertThat(actualResult.isComplete(), is(false)); - assertThat(actualResult.getInformationContext(), is(expectedResult.getInformationContext())); + assertThat(actualResult.complete(), is(false)); + assertThat(actualResult.informationContext(), is(expectedResult.informationContext())); } public void testExecuteWithPendingShards() { @@ -129,8 +129,8 @@ public void testExecuteWithPendingShards() { ); Result actualResult = step.isConditionMet(index, clusterState); - assertThat(actualResult.isComplete(), is(false)); - assertThat(actualResult.getInformationContext(), is(expectedResult.getInformationContext())); + assertThat(actualResult.complete(), is(false)); + assertThat(actualResult.informationContext(), is(expectedResult.informationContext())); } public void testExecuteWithPendingShardsAndTargetRoleNotPresentInCluster() { @@ -163,8 +163,8 @@ public void testExecuteWithPendingShardsAndTargetRoleNotPresentInCluster() { ); Result actualResult = step.isConditionMet(index, clusterState); - assertThat(actualResult.isComplete(), is(false)); - assertThat(actualResult.getInformationContext(), is(expectedResult.getInformationContext())); + assertThat(actualResult.complete(), is(false)); + assertThat(actualResult.informationContext(), is(expectedResult.informationContext())); } public void testExecuteIndexMissing() { @@ -174,8 +174,8 @@ public void testExecuteIndexMissing() { DataTierMigrationRoutedStep step = createRandomInstance(); Result actualResult = step.isConditionMet(index, clusterState); - assertThat(actualResult.isComplete(), is(false)); - assertThat(actualResult.getInformationContext(), is(nullValue())); + assertThat(actualResult.complete(), is(false)); + assertThat(actualResult.informationContext(), is(nullValue())); } public void testExecuteIsComplete() { @@ -199,8 +199,8 @@ public void testExecuteIsComplete() { .build(); DataTierMigrationRoutedStep step = createRandomInstance(); Result result = step.isConditionMet(index, clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), is(nullValue())); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), is(nullValue())); } public void testExecuteWithGenericDataNodes() { @@ -220,8 +220,8 @@ public void testExecuteWithGenericDataNodes() { .build(); DataTierMigrationRoutedStep step = createRandomInstance(); Result result = step.isConditionMet(index, clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), is(nullValue())); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), is(nullValue())); } public void testExecuteForIndexWithoutTierRoutingInformationWaitsForReplicasToBeActive() { @@ -245,8 +245,8 @@ public void testExecuteForIndexWithoutTierRoutingInformationWaitsForReplicasToBe Result expectedResult = new Result(false, waitingForActiveShardsAllocationInfo(1)); Result result = step.isConditionMet(index, clusterState); - assertThat(result.isComplete(), is(false)); - assertThat(result.getInformationContext(), is(expectedResult.getInformationContext())); + assertThat(result.complete(), is(false)); + assertThat(result.informationContext(), is(expectedResult.informationContext())); } { @@ -266,8 +266,8 @@ public void testExecuteForIndexWithoutTierRoutingInformationWaitsForReplicasToBe DataTierMigrationRoutedStep step = createRandomInstance(); Result result = step.isConditionMet(index, clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), is(nullValue())); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), is(nullValue())); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SegmentCountStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SegmentCountStepInfoTests.java index 7aeeba557ee54..9e0c7c7c6b167 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SegmentCountStepInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SegmentCountStepInfoTests.java @@ -38,7 +38,7 @@ public final void testEqualsAndHashcode() { } protected final Info copyInstance(Info instance) throws IOException { - return new Info(instance.getNumberShardsLeftToMerge()); + return new Info(instance.numberShardsLeftToMerge()); } protected Info mutateInstance(Info instance) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java index 59eff971c1643..592d259f07069 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java @@ -94,8 +94,8 @@ public void testConditionMet() { .build(); Result result = step.isConditionMet(originalIndexMetadata.getIndex(), clusterState); - assertTrue(result.isComplete()); - assertNull(result.getInformationContext()); + assertTrue(result.complete()); + assertNull(result.informationContext()); } public void testConditionNotMetBecauseOfActive() { @@ -137,8 +137,8 @@ public void testConditionNotMetBecauseOfActive() { .build(); Result result = step.isConditionMet(originalIndexMetadata.getIndex(), clusterState); - assertFalse(result.isComplete()); - assertEquals(new ShrunkShardsAllocatedStep.Info(true, shrinkNumberOfShards, false), result.getInformationContext()); + assertFalse(result.complete()); + assertEquals(new ShrunkShardsAllocatedStep.Info(true, shrinkNumberOfShards, false), result.informationContext()); } public void testConditionNotMetBecauseOfShrunkIndexDoesntExistYet() { @@ -166,7 +166,7 @@ public void testConditionNotMetBecauseOfShrunkIndexDoesntExistYet() { .build(); Result result = step.isConditionMet(originalIndexMetadata.getIndex(), clusterState); - assertFalse(result.isComplete()); - assertEquals(new ShrunkShardsAllocatedStep.Info(false, -1, false), result.getInformationContext()); + assertFalse(result.complete()); + assertEquals(new ShrunkShardsAllocatedStep.Info(false, -1, false), result.informationContext()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkenIndexCheckStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkenIndexCheckStepTests.java index 523404a00a0c5..4eb49df7f89c5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkenIndexCheckStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkenIndexCheckStepTests.java @@ -59,8 +59,8 @@ public void testConditionMet() { ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertTrue(result.isComplete()); - assertNull(result.getInformationContext()); + assertTrue(result.complete()); + assertNull(result.informationContext()); } public void testConditionNotMetBecauseNotSameShrunkenIndex() { @@ -77,8 +77,8 @@ public void testConditionNotMetBecauseNotSameShrunkenIndex() { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); Result result = step.isConditionMet(shrinkIndexMetadata.getIndex(), clusterState); - assertFalse(result.isComplete()); - assertEquals(new ShrunkenIndexCheckStep.Info(sourceIndex), result.getInformationContext()); + assertFalse(result.complete()); + assertEquals(new ShrunkenIndexCheckStep.Info(sourceIndex), result.informationContext()); } public void testConditionNotMetBecauseSourceIndexExists() { @@ -101,8 +101,8 @@ public void testConditionNotMetBecauseSourceIndexExists() { .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); Result result = step.isConditionMet(shrinkIndexMetadata.getIndex(), clusterState); - assertFalse(result.isComplete()); - assertEquals(new ShrunkenIndexCheckStep.Info(sourceIndex), result.getInformationContext()); + assertFalse(result.complete()); + assertEquals(new ShrunkenIndexCheckStep.Info(sourceIndex), result.informationContext()); } public void testIllegalState() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java index e12bae3b92f80..328698254dc76 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java @@ -125,7 +125,7 @@ public void testResultEvaluatedOnWriteIndexAliasWhenExists() { assertThat( "the rolled index has both the primary and the replica shards started so the condition should be met", - createRandomInstance().isConditionMet(originalIndex.getIndex(), clusterState).isComplete(), + createRandomInstance().isConditionMet(originalIndex.getIndex(), clusterState).complete(), is(true) ); } @@ -163,7 +163,7 @@ public void testResultEvaluatedOnOnlyIndexTheAliasPointsToIfWriteIndexIsNull() { assertThat( "the index the alias is pointing to has both the primary and the replica shards started so the condition should be" + " met", - createRandomInstance().isConditionMet(originalIndex.getIndex(), clusterState).isComplete(), + createRandomInstance().isConditionMet(originalIndex.getIndex(), clusterState).complete(), is(true) ); } @@ -244,13 +244,13 @@ public void testResultEvaluatedOnDataStream() throws IOException { boolean useFailureStore = randomBoolean(); IndexMetadata indexToOperateOn = useFailureStore ? failureOriginalIndexMeta : originalIndexMeta; ClusterStateWaitStep.Result result = waitForActiveShardsStep.isConditionMet(indexToOperateOn.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); + assertThat(result.complete(), is(false)); XContentBuilder expected = new WaitForActiveShardsStep.ActiveShardsInfo(2, "3", false).toXContent( JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS ); - String actualResultAsString = Strings.toString(result.getInformationContext()); + String actualResultAsString = Strings.toString(result.informationContext()); assertThat(actualResultAsString, is(Strings.toString(expected))); assertThat(actualResultAsString, containsString("waiting for [3] shards to become active, but only [2] are active")); } @@ -288,13 +288,13 @@ public void testResultReportsMeaningfulMessage() throws IOException { .build(); ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(originalIndex.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); + assertThat(result.complete(), is(false)); XContentBuilder expected = new WaitForActiveShardsStep.ActiveShardsInfo(2, "3", false).toXContent( JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS ); - String actualResultAsString = Strings.toString(result.getInformationContext()); + String actualResultAsString = Strings.toString(result.informationContext()); assertThat(actualResultAsString, is(Strings.toString(expected))); assertThat(actualResultAsString, containsString("waiting for [3] shards to become active, but only [2] are active")); } @@ -316,9 +316,9 @@ public void testResultReportsErrorMessage() { WaitForActiveShardsStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(new Index("index-000000", UUID.randomUUID().toString()), clusterState); - assertThat(result.isComplete(), is(false)); + assertThat(result.complete(), is(false)); - String actualResultAsString = Strings.toString(result.getInformationContext()); + String actualResultAsString = Strings.toString(result.informationContext()); assertThat( actualResultAsString, containsString( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForDataTierStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForDataTierStepTests.java index 3247c02cd9bac..00012575ea5de 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForDataTierStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForDataTierStepTests.java @@ -79,11 +79,11 @@ public void testConditionMet() { private void verify(WaitForDataTierStep step, ClusterState state, boolean complete, String message) { ClusterStateWaitStep.Result result = step.isConditionMet(null, state); - assertThat(result.isComplete(), is(complete)); + assertThat(result.complete(), is(complete)); if (message != null) { - assertThat(Strings.toString(result.getInformationContext()), containsString(message)); + assertThat(Strings.toString(result.informationContext()), containsString(message)); } else { - assertThat(result.getInformationContext(), is(nullValue())); + assertThat(result.informationContext(), is(nullValue())); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepInfoTests.java index 62c12e272ef59..0e5323d51f155 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepInfoTests.java @@ -10,7 +10,7 @@ import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ilm.WaitForFollowShardTasksStep.Info; -import org.elasticsearch.xpack.core.ilm.WaitForFollowShardTasksStep.Info.ShardFollowTaskInfo; +import org.elasticsearch.xpack.core.ilm.WaitForFollowShardTasksStep.ShardFollowTaskInfo; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepTests.java index 162f0ec3361b4..4ac5511a247c9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForFollowShardTasksStepTests.java @@ -131,10 +131,10 @@ public void onFailure(Exception e) { assertThat(informationContextHolder[0], notNullValue()); assertThat(exceptionHolder[0], nullValue()); WaitForFollowShardTasksStep.Info info = (WaitForFollowShardTasksStep.Info) informationContextHolder[0]; - assertThat(info.getShardFollowTaskInfos().size(), equalTo(1)); - assertThat(info.getShardFollowTaskInfos().get(0).getShardId(), equalTo(1)); - assertThat(info.getShardFollowTaskInfos().get(0).getLeaderGlobalCheckpoint(), equalTo(8L)); - assertThat(info.getShardFollowTaskInfos().get(0).getFollowerGlobalCheckpoint(), equalTo(3L)); + assertThat(info.shardFollowTaskInfos().size(), equalTo(1)); + assertThat(info.shardFollowTaskInfos().get(0).shardId(), equalTo(1)); + assertThat(info.shardFollowTaskInfos().get(0).leaderGlobalCheckpoint(), equalTo(8L)); + assertThat(info.shardFollowTaskInfos().get(0).followerGlobalCheckpoint(), equalTo(3L)); } public void testConditionNotMetNotAFollowerIndex() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java index 0ae7b02c7400a..1414788f3ff98 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java @@ -93,8 +93,8 @@ public void testConditionMetForGreen() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); } public void testConditionNotMetForGreen() { @@ -119,10 +119,10 @@ public void testConditionNotMetForGreen() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat(info, notNullValue()); - assertThat(info.getMessage(), equalTo("index is not green; not all shards are active")); + assertThat(info.message(), equalTo("index is not green; not all shards are active")); } public void testConditionNotMetNoIndexRoutingTable() { @@ -139,10 +139,10 @@ public void testConditionNotMetNoIndexRoutingTable() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat(info, notNullValue()); - assertThat(info.getMessage(), equalTo("index is red; no indexRoutingTable")); + assertThat(info.message(), equalTo("index is red; no indexRoutingTable")); } public void testConditionMetForYellow() { @@ -167,8 +167,8 @@ public void testConditionMetForYellow() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); } public void testConditionNotMetForYellow() { @@ -193,10 +193,10 @@ public void testConditionNotMetForYellow() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat(info, notNullValue()); - assertThat(info.getMessage(), equalTo("index is red; not all primary shards are active")); + assertThat(info.message(), equalTo("index is red; not all primary shards are active")); } public void testConditionNotMetNoIndexRoutingTableForYellow() { @@ -213,10 +213,10 @@ public void testConditionNotMetNoIndexRoutingTableForYellow() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); assertThat(info, notNullValue()); - assertThat(info.getMessage(), equalTo("index is red; no indexRoutingTable")); + assertThat(info.message(), equalTo("index is red; no indexRoutingTable")); } public void testStepReturnsFalseIfTargetIndexIsMissing() { @@ -243,11 +243,11 @@ public void testStepReturnsFalseIfTargetIndexIsMissing() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN, indexPrefix); ClusterStateWaitStep.Result result = step.isConditionMet(originalIndex.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); String targetIndex = indexPrefix + originalIndex.getIndex().getName(); assertThat( - info.getMessage(), + info.message(), is( "[" + step.getKey().action() @@ -303,9 +303,9 @@ public void testStepWaitsForTargetIndexHealthWhenPrefixConfigured() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN); ClusterStateWaitStep.Result result = step.isConditionMet(originalIndex.getIndex(), clusterTargetInitializing); - assertThat(result.isComplete(), is(false)); - SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInformationContext(); - assertThat(info.getMessage(), is("index is not green; not all shards are active")); + assertThat(result.complete(), is(false)); + SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.informationContext(); + assertThat(info.message(), is("index is not green; not all shards are active")); } { @@ -326,8 +326,8 @@ public void testStepWaitsForTargetIndexHealthWhenPrefixConfigured() { WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN); ClusterStateWaitStep.Result result = step.isConditionMet(originalIndex.getIndex(), clusterTargetInitializing); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexingCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexingCompleteStepTests.java index ad5e4c9533c99..2f91393b451d7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexingCompleteStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexingCompleteStepTests.java @@ -65,8 +65,8 @@ public void testConditionMet() { WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); } public void testConditionMetNotAFollowerIndex() { @@ -82,8 +82,8 @@ public void testConditionMetNotAFollowerIndex() { WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(true)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(true)); + assertThat(result.informationContext(), nullValue()); } public void testConditionNotMet() { @@ -104,10 +104,10 @@ public void testConditionNotMet() { WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState); - assertThat(result.isComplete(), is(false)); - assertThat(result.getInformationContext(), notNullValue()); + assertThat(result.complete(), is(false)); + assertThat(result.informationContext(), notNullValue()); WaitForIndexingCompleteStep.IndexingNotCompleteInfo info = (WaitForIndexingCompleteStep.IndexingNotCompleteInfo) result - .getInformationContext(); + .informationContext(); assertThat( info.getMessage(), equalTo( @@ -122,7 +122,7 @@ public void testIndexDeleted() { WaitForIndexingCompleteStep step = createRandomInstance(); ClusterStateWaitStep.Result result = step.isConditionMet(new Index("this-index-doesnt-exist", "uuid"), clusterState); - assertThat(result.isComplete(), is(false)); - assertThat(result.getInformationContext(), nullValue()); + assertThat(result.complete(), is(false)); + assertThat(result.informationContext(), nullValue()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationRoutedStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationRoutedStepInfoTests.java index 67214868293ea..0e6903ba6cf44 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationRoutedStepInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/step/info/AllocationRoutedStepInfoTests.java @@ -38,18 +38,18 @@ public final void testEqualsAndHashcode() { protected final AllocationInfo copyInstance(AllocationInfo instance) { return new AllocationInfo( - instance.getNumberOfReplicas(), - instance.getNumberShardsLeftToAllocate(), + instance.numberOfReplicas(), + instance.numberShardsLeftToAllocate(), instance.allShardsActive(), - instance.getMessage() + instance.message() ); } protected AllocationInfo mutateInstance(AllocationInfo instance) throws IOException { - long actualReplicas = instance.getNumberOfReplicas(); - long shardsToAllocate = instance.getNumberShardsLeftToAllocate(); + long actualReplicas = instance.numberOfReplicas(); + long shardsToAllocate = instance.numberShardsLeftToAllocate(); boolean allShardsActive = instance.allShardsActive(); - var message = instance.getMessage(); + var message = instance.message(); switch (between(0, 2)) { case 0 -> shardsToAllocate += between(1, 20); case 1 -> allShardsActive = allShardsActive == false; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index e06c7bc2708ca..9efe46402428c 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -43,7 +43,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.SortedMap; import java.util.TreeMap; @@ -811,13 +810,12 @@ static String convertAttributeValueToTierPreference(String nodeAttributeValue) { * Represents the elasticsearch abstractions that were, in some way, migrated such that the system is managing indices lifecycles and * allocations using data tiers. */ - public static final class MigratedEntities { - @Nullable - public final String removedIndexTemplateName; - public final List migratedIndices; - public final List migratedPolicies; - public final MigratedTemplates migratedTemplates; - + public record MigratedEntities( + @Nullable String removedIndexTemplateName, + List migratedIndices, + List migratedPolicies, + MigratedTemplates migratedTemplates + ) { public MigratedEntities( @Nullable String removedIndexTemplateName, List migratedIndices, @@ -830,36 +828,17 @@ public MigratedEntities( this.migratedTemplates = migratedTemplates; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MigratedEntities that = (MigratedEntities) o; - return Objects.equals(removedIndexTemplateName, that.removedIndexTemplateName) - && Objects.equals(migratedIndices, that.migratedIndices) - && Objects.equals(migratedPolicies, that.migratedPolicies) - && Objects.equals(migratedTemplates, that.migratedTemplates); - } - - @Override - public int hashCode() { - return Objects.hash(removedIndexTemplateName, migratedIndices, migratedPolicies, migratedTemplates); - } } /** * Represents the legacy, composable, and component templates that were migrated away from shard allocation settings based on custom * node attributes. */ - public static final class MigratedTemplates { - public final List migratedLegacyTemplates; - public final List migratedComposableTemplates; - public final List migratedComponentTemplates; - + public record MigratedTemplates( + List migratedLegacyTemplates, + List migratedComposableTemplates, + List migratedComponentTemplates + ) { public MigratedTemplates( List migratedLegacyTemplates, List migratedComposableTemplates, @@ -869,24 +848,5 @@ public MigratedTemplates( this.migratedComposableTemplates = Collections.unmodifiableList(migratedComposableTemplates); this.migratedComponentTemplates = Collections.unmodifiableList(migratedComponentTemplates); } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MigratedTemplates that = (MigratedTemplates) o; - return Objects.equals(migratedLegacyTemplates, that.migratedLegacyTemplates) - && Objects.equals(migratedComposableTemplates, that.migratedComposableTemplates) - && Objects.equals(migratedComponentTemplates, that.migratedComponentTemplates); - } - - @Override - public int hashCode() { - return Objects.hash(migratedLegacyTemplates, migratedComposableTemplates, migratedComponentTemplates); - } } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTask.java index 77b143f93576b..8c08194b11e05 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTask.java @@ -159,7 +159,7 @@ public ClusterState doExecute(final ClusterState currentState) throws IOExceptio // to be met (eg. {@link LifecycleSettings#LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING}, so it's important we // re-evaluate what the next step is after we evaluate the condition nextStepKey = currentStep.getNextStepKey(); - if (result.isComplete()) { + if (result.complete()) { logger.trace( "[{}] cluster state step condition met successfully ({}) [{}], moving to next step {}", index.getName(), @@ -180,7 +180,7 @@ public ClusterState doExecute(final ClusterState currentState) throws IOExceptio ); } } else { - final ToXContentObject stepInfo = result.getInformationContext(); + final ToXContentObject stepInfo = result.informationContext(); if (logger.isTraceEnabled()) { logger.trace( "[{}] condition not met ({}) [{}], returning existing state (info: {})", diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java index 494f0ee444236..ef7554beed9e9 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java @@ -100,12 +100,12 @@ protected void masterOperation( ).v2(); listener.onResponse( new MigrateToDataTiersResponse( - entities.removedIndexTemplateName, - entities.migratedPolicies, - entities.migratedIndices, - entities.migratedTemplates.migratedLegacyTemplates, - entities.migratedTemplates.migratedComposableTemplates, - entities.migratedTemplates.migratedComponentTemplates, + entities.removedIndexTemplateName(), + entities.migratedPolicies(), + entities.migratedIndices(), + entities.migratedTemplates().migratedLegacyTemplates(), + entities.migratedTemplates().migratedComposableTemplates(), + entities.migratedTemplates().migratedComponentTemplates(), true ) ); @@ -161,12 +161,12 @@ public void onFailure(Exception e) { MigratedEntities entities = migratedEntities.get(); listener.onResponse( new MigrateToDataTiersResponse( - entities.removedIndexTemplateName, - entities.migratedPolicies, - entities.migratedIndices, - entities.migratedTemplates.migratedLegacyTemplates, - entities.migratedTemplates.migratedComposableTemplates, - entities.migratedTemplates.migratedComponentTemplates, + entities.removedIndexTemplateName(), + entities.migratedPolicies(), + entities.migratedIndices(), + entities.migratedTemplates().migratedLegacyTemplates(), + entities.migratedTemplates().migratedComposableTemplates(), + entities.migratedTemplates().migratedComponentTemplates(), false ) ); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java index 51df651ea4a4c..570c2f5231acf 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java @@ -1080,11 +1080,11 @@ public void testMigrateToDataTiersRouting() { ); MigratedEntities migratedEntities = migratedEntitiesTuple.v2(); - assertThat(migratedEntities.removedIndexTemplateName, is("catch-all")); - assertThat(migratedEntities.migratedPolicies.size(), is(1)); - assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); - assertThat(migratedEntities.migratedIndices.size(), is(2)); - assertThat(migratedEntities.migratedIndices, hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); + assertThat(migratedEntities.removedIndexTemplateName(), is("catch-all")); + assertThat(migratedEntities.migratedPolicies().size(), is(1)); + assertThat(migratedEntities.migratedPolicies().get(0), is(lifecycleName)); + assertThat(migratedEntities.migratedIndices().size(), is(2)); + assertThat(migratedEntities.migratedIndices(), hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); ClusterState newState = migratedEntitiesTuple.v1(); assertThat(newState.metadata().getTemplates().size(), is(1)); @@ -1105,11 +1105,11 @@ public void testMigrateToDataTiersRouting() { ); MigratedEntities migratedEntities = migratedEntitiesTuple.v2(); - assertThat(migratedEntities.removedIndexTemplateName, nullValue()); - assertThat(migratedEntities.migratedPolicies.size(), is(1)); - assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); - assertThat(migratedEntities.migratedIndices.size(), is(2)); - assertThat(migratedEntities.migratedIndices, hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); + assertThat(migratedEntities.removedIndexTemplateName(), nullValue()); + assertThat(migratedEntities.migratedPolicies().size(), is(1)); + assertThat(migratedEntities.migratedPolicies().get(0), is(lifecycleName)); + assertThat(migratedEntities.migratedIndices().size(), is(2)); + assertThat(migratedEntities.migratedIndices(), hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); ClusterState newState = migratedEntitiesTuple.v1(); assertThat(newState.metadata().getTemplates().size(), is(2)); @@ -1130,10 +1130,10 @@ public void testMigrateToDataTiersRouting() { ); MigratedEntities migratedEntities = migratedEntitiesTuple.v2(); - assertThat(migratedEntities.migratedPolicies.size(), is(1)); - assertThat(migratedEntities.migratedPolicies.get(0), is(lifecycleName)); - assertThat(migratedEntities.migratedIndices.size(), is(2)); - assertThat(migratedEntities.migratedIndices, hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); + assertThat(migratedEntities.migratedPolicies().size(), is(1)); + assertThat(migratedEntities.migratedPolicies().get(0), is(lifecycleName)); + assertThat(migratedEntities.migratedIndices().size(), is(2)); + assertThat(migratedEntities.migratedIndices(), hasItems("indexWithWarmDataAttribute", "indexWithUnknownDataAttribute")); IndexMetadata migratedIndex; migratedIndex = migratedEntitiesTuple.v1().metadata().index("indexWithWarmDataAttribute"); @@ -1185,9 +1185,9 @@ public void testMigrateToDataTiersRoutingRequiresILMStopped() { null, false ); - assertThat(migratedState.v2().migratedIndices, empty()); - assertThat(migratedState.v2().migratedPolicies, empty()); - assertThat(migratedState.v2().removedIndexTemplateName, nullValue()); + assertThat(migratedState.v2().migratedIndices(), empty()); + assertThat(migratedState.v2().migratedPolicies(), empty()); + assertThat(migratedState.v2().removedIndexTemplateName(), nullValue()); } } @@ -1232,7 +1232,7 @@ public void testMigrationDoesNotRemoveComposableTemplates() { null, false ); - assertThat(migratedEntitiesTuple.v2().removedIndexTemplateName, nullValue()); + assertThat(migratedEntitiesTuple.v2().removedIndexTemplateName(), nullValue()); // the composable template still exists, however it was migrated to not use the custom require.data routing setting assertThat(migratedEntitiesTuple.v1().metadata().templatesV2().get(composableTemplateName), is(notNullValue())); } @@ -1676,9 +1676,9 @@ public void testMigrateIndexAndComponentTemplates() { Metadata.Builder mb = Metadata.builder(clusterState.metadata()); MetadataMigrateToDataTiersRoutingService.MigratedTemplates migratedTemplates = MetadataMigrateToDataTiersRoutingService .migrateIndexAndComponentTemplates(mb, clusterState, nodeAttrName); - assertThat(migratedTemplates.migratedLegacyTemplates, is(List.of("template-with-require-routing"))); - assertThat(migratedTemplates.migratedComposableTemplates, is(List.of("composable-template-with-require-routing"))); - assertThat(migratedTemplates.migratedComponentTemplates, is(List.of("component-with-require-and-include-routing"))); + assertThat(migratedTemplates.migratedLegacyTemplates(), is(List.of("template-with-require-routing"))); + assertThat(migratedTemplates.migratedComposableTemplates(), is(List.of("composable-template-with-require-routing"))); + assertThat(migratedTemplates.migratedComponentTemplates(), is(List.of("component-with-require-and-include-routing"))); } private String getWarmPhaseDef() { From f4dc7168eb790e4c4b0540b2ea5b7b9072840ffe Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:03:36 -0500 Subject: [PATCH 02/24] [ML] Checking for presence of error object when validating response (#118375) * Refactoring error handling logic * Refactoring base response handler to remove duplication * Update docs/changelog/118375.yaml * Addressing feedback --------- Co-authored-by: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> --- docs/changelog/118375.yaml | 5 + .../AlibabaCloudSearchResponseHandler.java | 14 +-- .../anthropic/AnthropicResponseHandler.java | 13 +- .../cohere/CohereResponseHandler.java | 14 +-- ...lasticInferenceServiceResponseHandler.java | 12 +- .../GoogleAiStudioResponseHandler.java | 13 +- .../GoogleVertexAiResponseHandler.java | 11 +- .../http/retry/BaseResponseHandler.java | 44 ++++++- .../external/http/retry/ErrorMessage.java | 12 -- .../external/http/retry/ErrorResponse.java | 37 ++++++ .../HuggingFaceResponseHandler.java | 14 +-- .../ibmwatsonx/IbmWatsonxResponseHandler.java | 13 +- .../openai/OpenAiResponseHandler.java | 19 +-- ...eMistralOpenAiExternalResponseHandler.java | 4 +- .../response/ErrorMessageResponseEntity.java | 39 +++--- ...AlibabaCloudSearchErrorResponseEntity.java | 19 +-- .../cohere/CohereErrorResponseEntity.java | 19 +-- ...icInferenceServiceErrorResponseEntity.java | 20 +-- .../GoogleAiStudioErrorResponseEntity.java | 24 ++-- .../GoogleVertexAiErrorResponseEntity.java | 24 ++-- .../HuggingFaceErrorResponseEntity.java | 20 +-- .../IbmWatsonxErrorResponseEntity.java | 22 ++-- .../openai/OpenAiErrorResponseEntity.java | 69 ---------- .../http/retry/BaseResponseHandlerTests.java | 118 ++++++++++++++++++ .../ErrorMessageResponseEntityTests.java | 33 ++++- ...baCloudSearchErrorResponseEntityTests.java | 2 +- .../CohereErrorResponseEntityTests.java | 8 +- ...erenceServiceErrorResponseEntityTests.java | 12 +- ...oogleAiStudioErrorResponseEntityTests.java | 9 +- ...oogleVertexAiErrorResponseEntityTests.java | 10 +- .../HuggingFaceErrorResponseEntityTests.java | 12 +- .../OpenAiErrorResponseEntityTests.java | 66 ---------- 32 files changed, 348 insertions(+), 403 deletions(-) create mode 100644 docs/changelog/118375.yaml delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorMessage.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorResponse.java delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntity.java delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntityTests.java diff --git a/docs/changelog/118375.yaml b/docs/changelog/118375.yaml new file mode 100644 index 0000000000000..bad2751aeaa50 --- /dev/null +++ b/docs/changelog/118375.yaml @@ -0,0 +1,5 @@ +pr: 118375 +summary: Check for presence of error object when validating streaming responses from integrations in the inference API +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java index ecfa988b5035e..30b371f3172ea 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.alibabacloudsearch; -import org.apache.logging.log4j.Logger; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; @@ -15,9 +14,6 @@ import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.alibabacloudsearch.AlibabaCloudSearchErrorResponseEntity; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; - -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; /** * Defines how to handle various errors returned from the AlibabaCloudSearch integration. @@ -28,13 +24,6 @@ public AlibabaCloudSearchResponseHandler(String requestType, ResponseParser pars super(requestType, parseFunction, AlibabaCloudSearchErrorResponseEntity::fromResponse); } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - /** * Validates the status code throws an RetryException if not in the range [200, 300). * @@ -42,7 +31,8 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @param result The http response and body * @throws RetryException Throws if status code is {@code >= 300 or < 200 } */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { int statusCode = result.response().getStatusLine().getStatusCode(); if (RestStatus.isSuccessful(statusCode)) { return; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java index 373045f879afa..d9a78a56af0d6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.anthropic; -import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; @@ -19,11 +18,9 @@ import org.elasticsearch.xpack.inference.external.response.ErrorMessageResponseEntity; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import java.util.concurrent.Flow; -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; import static org.elasticsearch.xpack.inference.external.http.retry.ResponseHandlerUtils.getFirstHeaderOrUnknown; public class AnthropicResponseHandler extends BaseResponseHandler { @@ -54,13 +51,6 @@ public AnthropicResponseHandler(String requestType, ResponseParser parseFunction this.canHandleStreamingResponses = canHandleStreamingResponses; } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - @Override public boolean canHandleStreamingResponses() { return canHandleStreamingResponses; @@ -83,7 +73,8 @@ public InferenceServiceResults parseResult(Request request, Flow.Publisher= 300 or < 200 } */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java index ac2e1747f8057..e3a74785caa4b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.cohere; -import org.apache.logging.log4j.Logger; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; @@ -17,12 +16,9 @@ import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.cohere.CohereErrorResponseEntity; import org.elasticsearch.xpack.inference.external.response.streaming.NewlineDelimitedByteProcessor; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import java.util.concurrent.Flow; -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; - /** * Defines how to handle various errors returned from the Cohere integration. * @@ -45,13 +41,6 @@ public CohereResponseHandler(String requestType, ResponseParser parseFunction, b this.canHandleStreamingResponse = canHandleStreamingResponse; } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - @Override public boolean canHandleStreamingResponses() { return canHandleStreamingResponse; @@ -73,7 +62,8 @@ public InferenceServiceResults parseResult(Request request, Flow.Publisher= 300 or < 200 } */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java index 2b79afb3b56fd..b11b4a743fb27 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.elastic; -import org.apache.logging.log4j.Logger; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; @@ -15,9 +14,6 @@ import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceErrorResponseEntity; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; - -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public class ElasticInferenceServiceResponseHandler extends BaseResponseHandler { @@ -26,13 +22,7 @@ public ElasticInferenceServiceResponseHandler(String requestType, ResponseParser } @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java index 0241dcd6142a6..d61e82cb83b45 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.googleaistudio; -import org.apache.logging.log4j.Logger; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xcontent.XContentParser; @@ -20,13 +19,11 @@ import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioErrorResponseEntity; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import java.io.IOException; import java.util.concurrent.Flow; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public class GoogleAiStudioResponseHandler extends BaseResponseHandler { @@ -52,13 +49,6 @@ public GoogleAiStudioResponseHandler( this.content = content; } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - /** * Validates the status code and throws a RetryException if not in the range [200, 300). * @@ -67,7 +57,8 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @param result The http response and body * @throws RetryException Throws if status code is {@code >= 300 or < 200 } */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java index 6b1aef9856d33..6924a5e4cb336 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java @@ -7,17 +7,14 @@ package org.elasticsearch.xpack.inference.external.googlevertexai; -import org.apache.logging.log4j.Logger; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.googlevertexai.GoogleVertexAiErrorResponseEntity; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public class GoogleVertexAiResponseHandler extends BaseResponseHandler { @@ -28,13 +25,7 @@ public GoogleVertexAiResponseHandler(String requestType, ResponseParser parseFun } @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java index c9cbe169ec03d..1b0dd893ada6f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java @@ -7,16 +7,20 @@ package org.elasticsearch.xpack.inference.external.http.retry; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.Strings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import java.util.Objects; import java.util.function.Function; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public abstract class BaseResponseHandler implements ResponseHandler { @@ -27,14 +31,15 @@ public abstract class BaseResponseHandler implements ResponseHandler { public static final String REDIRECTION = "Unhandled redirection"; public static final String CONTENT_TOO_LARGE = "Received a content too large status code"; public static final String UNSUCCESSFUL = "Received an unsuccessful status code"; + public static final String SERVER_ERROR_OBJECT = "Received an error response"; public static final String BAD_REQUEST = "Received a bad request status code"; public static final String METHOD_NOT_ALLOWED = "Received a method not allowed status code"; protected final String requestType; private final ResponseParser parseFunction; - private final Function errorParseFunction; + private final Function errorParseFunction; - public BaseResponseHandler(String requestType, ResponseParser parseFunction, Function errorParseFunction) { + public BaseResponseHandler(String requestType, ResponseParser parseFunction, Function errorParseFunction) { this.requestType = Objects.requireNonNull(requestType); this.parseFunction = Objects.requireNonNull(parseFunction); this.errorParseFunction = Objects.requireNonNull(errorParseFunction); @@ -54,11 +59,42 @@ public String getRequestType() { return requestType; } + @Override + public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) { + checkForFailureStatusCode(request, result); + checkForEmptyBody(throttlerManager, logger, request, result); + + // When the response is streamed the status code could be 200 but the error object will be set + // so we need to check for that specifically + checkForErrorObject(request, result); + } + + protected abstract void checkForFailureStatusCode(Request request, HttpResult result); + + private void checkForErrorObject(Request request, HttpResult result) { + var errorEntity = errorParseFunction.apply(result); + + if (errorEntity.errorStructureFound()) { + // We don't really know what happened because the status code was 200 so we'll return a failure and let the + // client retry if necessary + // If we did want to retry here, we'll need to determine if this was a streaming request, if it was + // we shouldn't retry because that would replay the entire streaming request and the client would get + // duplicate chunks back + throw new RetryException(false, buildError(SERVER_ERROR_OBJECT, request, result, errorEntity)); + } + } + protected Exception buildError(String message, Request request, HttpResult result) { var errorEntityMsg = errorParseFunction.apply(result); + return buildError(message, request, result, errorEntityMsg); + } + + protected Exception buildError(String message, Request request, HttpResult result, ErrorResponse errorResponse) { var responseStatusCode = result.response().getStatusLine().getStatusCode(); - if (errorEntityMsg == null) { + if (errorResponse == null + || errorResponse.errorStructureFound() == false + || Strings.isNullOrEmpty(errorResponse.getErrorMessage())) { return new ElasticsearchStatusException( format( "%s for request from inference entity id [%s] status [%s]", @@ -76,7 +112,7 @@ protected Exception buildError(String message, Request request, HttpResult resul message, request.getInferenceEntityId(), responseStatusCode, - errorEntityMsg.getErrorMessage() + errorResponse.getErrorMessage() ), toRestStatus(responseStatusCode) ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorMessage.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorMessage.java deleted file mode 100644 index a4be7f15827fb..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorMessage.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.http.retry; - -public interface ErrorMessage { - String getErrorMessage(); -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorResponse.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorResponse.java new file mode 100644 index 0000000000000..be9669c331371 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/ErrorResponse.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.retry; + +import java.util.Objects; + +public class ErrorResponse { + + // Denotes an error object that was not found + public static final ErrorResponse UNDEFINED_ERROR = new ErrorResponse(false); + + private final String errorMessage; + private final boolean errorStructureFound; + + public ErrorResponse(String errorMessage) { + this.errorMessage = Objects.requireNonNull(errorMessage); + this.errorStructureFound = true; + } + + private ErrorResponse(boolean errorStructureFound) { + this.errorMessage = ""; + this.errorStructureFound = errorStructureFound; + } + + public String getErrorMessage() { + return errorMessage; + } + + public boolean errorStructureFound() { + return errorStructureFound; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java index f6fd9afabe28d..5b6b0de4f2428 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.huggingface; -import org.apache.logging.log4j.Logger; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; @@ -15,9 +14,6 @@ import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.huggingface.HuggingFaceErrorResponseEntity; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; - -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public class HuggingFaceResponseHandler extends BaseResponseHandler { @@ -25,13 +21,6 @@ public HuggingFaceResponseHandler(String requestType, ResponseParser parseFuncti super(requestType, parseFunction, HuggingFaceErrorResponseEntity::fromResponse); } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - /** * Validates the status code and throws a RetryException if it is not in the range [200, 300). * @@ -40,7 +29,8 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @param result the http response and body * @throws RetryException thrown if status code is {@code >= 300 or < 200} */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java index cb686ddb654db..6d1d3fb2a4f91 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java @@ -7,17 +7,14 @@ package org.elasticsearch.xpack.inference.external.ibmwatsonx; -import org.apache.logging.log4j.Logger; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.ibmwatsonx.IbmWatsonxErrorResponseEntity; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; public class IbmWatsonxResponseHandler extends BaseResponseHandler { @@ -25,13 +22,6 @@ public IbmWatsonxResponseHandler(String requestType, ResponseParser parseFunctio super(requestType, parseFunction, IbmWatsonxErrorResponseEntity::fromResponse); } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - /** * Validates the status code and throws a RetryException if it is not in the range [200, 300). * @@ -41,7 +31,8 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @param result the http response and body * @throws RetryException thrown if status code is {@code >= 300 or < 200} */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java index 6404236d51184..cf867fb1a0ab0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.openai; -import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; @@ -17,14 +16,12 @@ import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; -import org.elasticsearch.xpack.inference.external.response.openai.OpenAiErrorResponseEntity; +import org.elasticsearch.xpack.inference.external.response.ErrorMessageResponseEntity; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; -import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import java.util.concurrent.Flow; -import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; import static org.elasticsearch.xpack.inference.external.http.retry.ResponseHandlerUtils.getFirstHeaderOrUnknown; public class OpenAiResponseHandler extends BaseResponseHandler { @@ -47,17 +44,10 @@ public class OpenAiResponseHandler extends BaseResponseHandler { private final boolean canHandleStreamingResponses; public OpenAiResponseHandler(String requestType, ResponseParser parseFunction, boolean canHandleStreamingResponses) { - super(requestType, parseFunction, OpenAiErrorResponseEntity::fromResponse); + super(requestType, parseFunction, ErrorMessageResponseEntity::fromResponse); this.canHandleStreamingResponses = canHandleStreamingResponses; } - @Override - public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) - throws RetryException { - checkForFailureStatusCode(request, result); - checkForEmptyBody(throttlerManager, logger, request, result); - } - /** * Validates the status code throws an RetryException if not in the range [200, 300). * @@ -66,7 +56,8 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @param result The http response and body * @throws RetryException Throws if status code is {@code >= 300 or < 200 } */ - void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { if (result.isSuccessfulResponse()) { return; } @@ -104,7 +95,7 @@ private static boolean isContentTooLarge(HttpResult result) { } if (statusCode == 400) { - var errorEntity = OpenAiErrorResponseEntity.fromResponse(result); + var errorEntity = ErrorMessageResponseEntity.fromResponse(result); return errorEntity != null && errorEntity.getErrorMessage().contains(CONTENT_TOO_LARGE_MESSAGE); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java index 7764a5b6586e2..de627a758c7c3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java @@ -14,7 +14,7 @@ import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.openai.OpenAiStreamingProcessor; @@ -54,7 +54,7 @@ public class AzureMistralOpenAiExternalResponseHandler extends BaseResponseHandl public AzureMistralOpenAiExternalResponseHandler( String requestType, ResponseParser parseFunction, - Function errorParseFunction, + Function errorParseFunction, boolean canHandleStreamingResponses ) { super(requestType, parseFunction, errorParseFunction); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntity.java index dbf2b37955b22..489844f4d14de 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntity.java @@ -12,9 +12,10 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import java.util.Map; +import java.util.Objects; /** * A pattern is emerging in how external providers provide error responses. @@ -31,27 +32,14 @@ * This currently covers error handling for Azure AI Studio, however this pattern * can be used to simplify and refactor handling for Azure OpenAI and OpenAI responses. */ -public class ErrorMessageResponseEntity implements ErrorMessage { - protected String errorMessage; +public class ErrorMessageResponseEntity extends ErrorResponse { public ErrorMessageResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; + super(errorMessage); } - @Override - public String getErrorMessage() { - return errorMessage; - } - - /** - * Standard error response parser. This can be overridden for those subclasses that - * might have a different format - * - * @param response the HttpResult - * @return the error response - */ @SuppressWarnings("unchecked") - public static ErrorMessage fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response, String defaultMessage) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -61,14 +49,23 @@ public static ErrorMessage fromResponse(HttpResult response) { var error = (Map) responseMap.get("error"); if (error != null) { var message = (String) error.get("message"); - if (message != null) { - return new ErrorMessageResponseEntity(message); - } + return new ErrorMessageResponseEntity(Objects.requireNonNullElse(message, defaultMessage)); } } catch (Exception e) { // swallow the error } - return null; + return ErrorResponse.UNDEFINED_ERROR; + } + + /** + * Standard error response parser. This can be overridden for those subclasses that + * might have a different format + * + * @param response the HttpResult + * @return the error response + */ + public static ErrorResponse fromResponse(HttpResult response) { + return fromResponse(response, ""); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java index 77a0c6ecc7cc8..fe69176db2a1d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java @@ -14,20 +14,13 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; -public class AlibabaCloudSearchErrorResponseEntity implements ErrorMessage { +public class AlibabaCloudSearchErrorResponseEntity extends ErrorResponse { private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchErrorResponseEntity.class); - private final String errorMessage; - private AlibabaCloudSearchErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public String getErrorMessage() { - return errorMessage; + super(errorMessage); } /** @@ -44,9 +37,9 @@ public String getErrorMessage() { * * @param response The error response * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the message field + * or {@link ErrorResponse#UNDEFINED_ERROR} if the message field wasn't found */ - public static AlibabaCloudSearchErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -64,6 +57,6 @@ public static AlibabaCloudSearchErrorResponseEntity fromResponse(HttpResult resp // swallow the error } - return null; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntity.java index 7d1731105e2f5..28f361b12fa2b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntity.java @@ -12,19 +12,12 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; -public class CohereErrorResponseEntity implements ErrorMessage { - - private final String errorMessage; +public class CohereErrorResponseEntity extends ErrorResponse { private CohereErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public String getErrorMessage() { - return errorMessage; + super(errorMessage); } /** @@ -38,9 +31,9 @@ public String getErrorMessage() { * * @param response The error response * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the message field + * or {@link ErrorResponse#UNDEFINED_ERROR} if the message field wasn't found */ - public static CohereErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -54,6 +47,6 @@ public static CohereErrorResponseEntity fromResponse(HttpResult response) { // swallow the error } - return null; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java index c860821c81bbf..696be7b2acdd2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java @@ -9,27 +9,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; -public class ElasticInferenceServiceErrorResponseEntity implements ErrorMessage { - - private final String errorMessage; +public class ElasticInferenceServiceErrorResponseEntity extends ErrorResponse { private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceErrorResponseEntity.class); private ElasticInferenceServiceErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public String getErrorMessage() { - return errorMessage; + super(errorMessage); } /** @@ -43,9 +35,9 @@ public String getErrorMessage() { * * @param response The error response * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the error field + * or {@link ErrorResponse#UNDEFINED_ERROR} if the error field wasn't found */ - public static @Nullable ElasticInferenceServiceErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -59,6 +51,6 @@ public String getErrorMessage() { logger.debug("Failed to parse error response", e); } - return null; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntity.java index f57f672e10b16..fa8751a2fb99b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntity.java @@ -12,21 +12,15 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import java.util.Map; +import java.util.Objects; -public class GoogleAiStudioErrorResponseEntity implements ErrorMessage { - - private final String errorMessage; +public class GoogleAiStudioErrorResponseEntity extends ErrorResponse { private GoogleAiStudioErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public String getErrorMessage() { - return errorMessage; + super(errorMessage); } /** @@ -52,11 +46,11 @@ public String getErrorMessage() { * * @param response The error response * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the `error.message` field + * or {@link ErrorResponse#UNDEFINED_ERROR} if the error field wasn't found */ @SuppressWarnings("unchecked") - public static GoogleAiStudioErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -65,14 +59,12 @@ public static GoogleAiStudioErrorResponseEntity fromResponse(HttpResult response var error = (Map) responseMap.get("error"); if (error != null) { var message = (String) error.get("message"); - if (message != null) { - return new GoogleAiStudioErrorResponseEntity(message); - } + return new GoogleAiStudioErrorResponseEntity(Objects.requireNonNullElse(message, "")); } } catch (Exception e) { // swallow the error } - return null; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntity.java index bf14d751db868..99e6f45f7e988 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntity.java @@ -12,21 +12,15 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import java.util.Map; +import java.util.Objects; -public class GoogleVertexAiErrorResponseEntity implements ErrorMessage { - - private final String errorMessage; +public class GoogleVertexAiErrorResponseEntity extends ErrorResponse { private GoogleVertexAiErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public String getErrorMessage() { - return errorMessage; + super(errorMessage); } /** @@ -54,10 +48,10 @@ public String getErrorMessage() { * * @param response The error response * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the `error.message` field + * or {@link ErrorResponse#UNDEFINED_ERROR} if the error field wasn't found */ @SuppressWarnings("unchecked") - public static GoogleVertexAiErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -66,14 +60,12 @@ public static GoogleVertexAiErrorResponseEntity fromResponse(HttpResult response var error = (Map) responseMap.get("error"); if (error != null) { var message = (String) error.get("message"); - if (message != null) { - return new GoogleVertexAiErrorResponseEntity(message); - } + return new GoogleVertexAiErrorResponseEntity(Objects.requireNonNullElse(message, "")); } } catch (Exception e) { // swallow the error } - return null; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntity.java index faeb7c6ac4fa9..af9057c334b12 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntity.java @@ -12,9 +12,14 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; + +public class HuggingFaceErrorResponseEntity extends ErrorResponse { + + public HuggingFaceErrorResponseEntity(String message) { + super(message); + } -public record HuggingFaceErrorResponseEntity(String message) implements ErrorMessage { /** * An example error response for invalid auth would look like * @@ -26,9 +31,9 @@ public record HuggingFaceErrorResponseEntity(String message) implements ErrorMes * * @param response The error response * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the error field + * or {@link ErrorResponse#UNDEFINED_ERROR} if the error field wasn't found */ - public static HuggingFaceErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -42,11 +47,6 @@ public static HuggingFaceErrorResponseEntity fromResponse(HttpResult response) { // swallow the error } - return null; - } - - @Override - public String getErrorMessage() { - return message; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxErrorResponseEntity.java index 582e3d009be50..9c805b22ff018 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxErrorResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxErrorResponseEntity.java @@ -12,25 +12,19 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import java.util.Map; +import java.util.Objects; -public class IbmWatsonxErrorResponseEntity implements ErrorMessage { - - private final String errorMessage; +public class IbmWatsonxErrorResponseEntity extends ErrorResponse { private IbmWatsonxErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public String getErrorMessage() { - return errorMessage; + super(errorMessage); } @SuppressWarnings("unchecked") - public static IbmWatsonxErrorResponseEntity fromResponse(HttpResult response) { + public static ErrorResponse fromResponse(HttpResult response) { try ( XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) .createParser(XContentParserConfiguration.EMPTY, response.body()) @@ -39,14 +33,12 @@ public static IbmWatsonxErrorResponseEntity fromResponse(HttpResult response) { var error = (Map) responseMap.get("error"); if (error != null) { var message = (String) error.get("message"); - if (message != null) { - return new IbmWatsonxErrorResponseEntity(message); - } + return new IbmWatsonxErrorResponseEntity(Objects.requireNonNullElse(message, "")); } } catch (Exception e) { // swallow the error } - return null; + return ErrorResponse.UNDEFINED_ERROR; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntity.java deleted file mode 100644 index a364be29ada33..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntity.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.response.openai; - -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage; - -import java.util.Map; - -public class OpenAiErrorResponseEntity implements ErrorMessage { - - private final String errorMessage; - - private OpenAiErrorResponseEntity(String errorMessage) { - this.errorMessage = errorMessage; - } - - public String getErrorMessage() { - return errorMessage; - } - - /** - * An example error response for invalid auth would look like - * - * { - * "error": { - * "message": "You didn't provide an API key...", - * "type": "invalid_request_error", - * "param": null, - * "code": null - * } - * } - * - * - * - * @param response The error response - * @return An error entity if the response is JSON with the above structure - * or null if the response does not contain the error.message field - */ - @SuppressWarnings("unchecked") - public static OpenAiErrorResponseEntity fromResponse(HttpResult response) { - try ( - XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) - .createParser(XContentParserConfiguration.EMPTY, response.body()) - ) { - var responseMap = jsonParser.map(); - var error = (Map) responseMap.get("error"); - if (error != null) { - var message = (String) error.get("message"); - if (message != null) { - return new OpenAiErrorResponseEntity(message); - } - } - } catch (Exception e) { - // swallow the error - } - - return null; - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandlerTests.java index b7095979b0fa5..444a187261fff 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandlerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandlerTests.java @@ -7,11 +7,22 @@ package org.elasticsearch.xpack.inference.external.http.retry; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.apache.logging.log4j.Logger; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.ErrorMessageResponseEntity; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; + +import java.nio.charset.StandardCharsets; import static org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler.toRestStatus; import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class BaseResponseHandlerTests extends ESTestCase { public void testToRestStatus_ReturnsBadRequest_WhenStatusIs500() { @@ -29,4 +40,111 @@ public void testToRestStatus_ReturnsStatusCodeValue_WhenStatusIs200() { public void testToRestStatus_ReturnsBadRequest_WhenStatusIsUnknown() { assertThat(toRestStatus(1000), is(RestStatus.BAD_REQUEST)); } + + public void testValidateResponse_DoesNotThrowAnExceptionWhenStatus200_AndNoErrorObject() { + var handler = getBaseResponseHandler(); + + String responseJson = """ + { + "field": "hello" + } + """; + + var response = mock200Response(); + + var request = mock(Request.class); + when(request.getInferenceEntityId()).thenReturn("abc"); + + handler.validateResponse( + mock(ThrottlerManager.class), + mock(Logger.class), + request, + new HttpResult(response, responseJson.getBytes(StandardCharsets.UTF_8)) + ); + } + + public void testValidateResponse_ThrowsErrorWhenMalformedErrorObjectExists() { + var handler = getBaseResponseHandler(); + + String responseJson = """ + { + "error": { + "type": "not_found_error" + } + } + """; + + var response = mock200Response(); + + var request = mock(Request.class); + when(request.getInferenceEntityId()).thenReturn("abc"); + + var exception = expectThrows( + RetryException.class, + () -> handler.validateResponse( + mock(ThrottlerManager.class), + mock(Logger.class), + request, + new HttpResult(response, responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertFalse(exception.shouldRetry()); + assertThat( + exception.getCause().getMessage(), + is("Received an error response for request from inference entity id [abc] status [200]") + ); + } + + public void testValidateResponse_ThrowsErrorWhenWellFormedErrorObjectExists() { + var handler = getBaseResponseHandler(); + + String responseJson = """ + { + "error": { + "type": "not_found_error", + "message": "a message" + } + } + """; + + var response = mock200Response(); + + var request = mock(Request.class); + when(request.getInferenceEntityId()).thenReturn("abc"); + + var exception = expectThrows( + RetryException.class, + () -> handler.validateResponse( + mock(ThrottlerManager.class), + mock(Logger.class), + request, + new HttpResult(response, responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertFalse(exception.shouldRetry()); + assertThat( + exception.getCause().getMessage(), + is("Received an error response for request from inference entity id [abc] status [200]. Error message: [a message]") + ); + } + + private static HttpResponse mock200Response() { + int statusCode = 200; + var statusLine = mock(StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + + var response = mock(HttpResponse.class); + when(response.getStatusLine()).thenReturn(statusLine); + + return response; + } + + private static BaseResponseHandler getBaseResponseHandler() { + return new BaseResponseHandler("abc", (Request request, HttpResult result) -> null, ErrorMessageResponseEntity::fromResponse) { + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) {} + }; + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntityTests.java index d57d1537f6c30..9ad9b9f3ca0a5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ErrorMessageResponseEntityTests.java @@ -11,10 +11,12 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import java.nio.charset.StandardCharsets; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; public class ErrorMessageResponseEntityTests extends ESTestCase { @@ -33,11 +35,29 @@ public void testErrorResponse_ExtractsError() { assertThat(error.getErrorMessage(), is("test_error_message")); } + public void testFromResponse_WithOtherFieldsPresent() { + String responseJson = """ + { + "error": { + "message": "You didn't provide an API key", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + """; + + ErrorResponse errorMessage = ErrorMessageResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + assertEquals("You didn't provide an API key", errorMessage.getErrorMessage()); + } + public void testFromResponse_noMessage() { String responseJson = """ { "error": { - "type": "not_found_error", + "type": "not_found_error" } } """; @@ -45,21 +65,22 @@ public void testFromResponse_noMessage() { var errorMessage = ErrorMessageResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertNull(errorMessage); + assertThat(errorMessage.getErrorMessage(), is("")); + assertTrue(errorMessage.errorStructureFound()); } - public void testErrorResponse_ReturnsNullIfNoError() { + public void testErrorResponse_ReturnsUndefinedObjectIfNoError() { var result = getMockResult(""" {"noerror":true}"""); var error = ErrorMessageResponseEntity.fromResponse(result); - assertNull(error); + assertThat(error, sameInstance(ErrorResponse.UNDEFINED_ERROR)); } - public void testErrorResponse_ReturnsNullIfNotJson() { + public void testErrorResponse_ReturnsUndefinedObjectIfNotJson() { var result = getMockResult("not a json string"); var error = ErrorMessageResponseEntity.fromResponse(result); - assertNull(error); + assertThat(error, sameInstance(ErrorResponse.UNDEFINED_ERROR)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java index a03349c66b6d5..1dd570537f12e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java @@ -26,7 +26,7 @@ public void testFromResponse() { } """; - AlibabaCloudSearchErrorResponseEntity errorMessage = AlibabaCloudSearchErrorResponseEntity.fromResponse( + var errorMessage = AlibabaCloudSearchErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertNotNull(errorMessage); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntityTests.java index a2b1c26b2b3d5..d770adb0b5a65 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereErrorResponseEntityTests.java @@ -10,7 +10,9 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; import java.nio.charset.StandardCharsets; @@ -25,7 +27,7 @@ public void testFromResponse() { } """; - CohereErrorResponseEntity errorMessage = CohereErrorResponseEntity.fromResponse( + var errorMessage = CohereErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertNotNull(errorMessage); @@ -42,9 +44,9 @@ public void testFromResponse_noMessage() { } """; - CohereErrorResponseEntity errorMessage = CohereErrorResponseEntity.fromResponse( + var errorMessage = CohereErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertNull(errorMessage); + assertThat(errorMessage, Matchers.sameInstance(ErrorResponse.UNDEFINED_ERROR)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java index 4da0518084828..d99e1c1ce5afc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java @@ -10,6 +10,8 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; +import org.hamcrest.Matchers; import java.nio.charset.StandardCharsets; @@ -25,7 +27,7 @@ public void testFromResponse() { } """; - ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( + var errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -40,11 +42,11 @@ public void testFromResponse_NoErrorMessagePresent() { } """; - ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( + var errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertNull(errorResponseEntity); + assertThat(errorResponseEntity, Matchers.sameInstance(ErrorResponse.UNDEFINED_ERROR)); } public void testFromResponse_InvalidJson() { @@ -52,10 +54,10 @@ public void testFromResponse_InvalidJson() { { """; - ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( + var errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), invalidResponseJson.getBytes(StandardCharsets.UTF_8)) ); - assertNull(errorResponseEntity); + assertThat(errorResponseEntity, Matchers.sameInstance(ErrorResponse.UNDEFINED_ERROR)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntityTests.java index 61448f2e35bdf..e2ccbb4167ef6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioErrorResponseEntityTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.inference.external.http.HttpResult; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; public class GoogleAiStudioErrorResponseEntityTests extends ESTestCase { @@ -48,7 +49,7 @@ public void testErrorResponse_ExtractsError() { assertThat(error.getErrorMessage(), is("error message")); } - public void testErrorResponse_ReturnsNullIfNoError() { + public void testErrorResponse_ReturnsUndefinedObjectIfNoError() { var result = getMockResult(""" { "foo": "bar" @@ -56,13 +57,13 @@ public void testErrorResponse_ReturnsNullIfNoError() { """); var error = GoogleAiStudioErrorResponseEntity.fromResponse(result); - assertNull(error); + assertThat(error, sameInstance(GoogleAiStudioErrorResponseEntity.UNDEFINED_ERROR)); } - public void testErrorResponse_ReturnsNullIfNotJson() { + public void testErrorResponse_ReturnsUndefinedIfNotJson() { var result = getMockResult("error message"); var error = GoogleAiStudioErrorResponseEntity.fromResponse(result); - assertNull(error); + assertThat(error, sameInstance(GoogleAiStudioErrorResponseEntity.UNDEFINED_ERROR)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntityTests.java index e2c9ebed2c164..23bb2f9829e62 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiErrorResponseEntityTests.java @@ -11,8 +11,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; public class GoogleVertexAiErrorResponseEntityTests extends ESTestCase { @@ -48,7 +50,7 @@ public void testErrorResponse_ExtractsError() { assertThat(error.getErrorMessage(), is("error message")); } - public void testErrorResponse_ReturnsNullIfNoError() { + public void testErrorResponse_ReturnsUndefinedObjectIfNoError() { var result = getMockResult(""" { "foo": "bar" @@ -56,14 +58,14 @@ public void testErrorResponse_ReturnsNullIfNoError() { """); var error = GoogleVertexAiErrorResponseEntity.fromResponse(result); - assertNull(error); + assertThat(error, sameInstance(ErrorResponse.UNDEFINED_ERROR)); } - public void testErrorResponse_ReturnsNullIfNotJson() { + public void testErrorResponse_ReturnsUndefinedObjectIfNotJson() { var result = getMockResult("error message"); var error = GoogleVertexAiErrorResponseEntity.fromResponse(result); - assertNull(error); + assertThat(error, sameInstance(ErrorResponse.UNDEFINED_ERROR)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntityTests.java index ed381de844731..06b15d57f5a89 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceErrorResponseEntityTests.java @@ -10,6 +10,8 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; +import org.hamcrest.Matchers; import java.nio.charset.StandardCharsets; @@ -23,7 +25,7 @@ public void testFromResponse() { } """; - HuggingFaceErrorResponseEntity errorMessage = HuggingFaceErrorResponseEntity.fromResponse( + var errorMessage = HuggingFaceErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertNotNull(errorMessage); @@ -39,10 +41,10 @@ public void testFromResponse_noMessage() { } """; - HuggingFaceErrorResponseEntity errorMessage = HuggingFaceErrorResponseEntity.fromResponse( + var errorMessage = HuggingFaceErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertNull(errorMessage); + assertThat(errorMessage, Matchers.sameInstance(ErrorResponse.UNDEFINED_ERROR)); } public void testFromResponse_noError() { @@ -54,9 +56,9 @@ public void testFromResponse_noError() { } """; - HuggingFaceErrorResponseEntity errorMessage = HuggingFaceErrorResponseEntity.fromResponse( + var errorMessage = HuggingFaceErrorResponseEntity.fromResponse( new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertNull(errorMessage); + assertThat(errorMessage, Matchers.sameInstance(ErrorResponse.UNDEFINED_ERROR)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntityTests.java deleted file mode 100644 index 4dc6c4190f92c..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiErrorResponseEntityTests.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.response.openai; - -import org.apache.http.HttpResponse; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.inference.external.http.HttpResult; - -import java.nio.charset.StandardCharsets; - -import static org.mockito.Mockito.mock; - -public class OpenAiErrorResponseEntityTests extends ESTestCase { - public void testFromResponse() { - String responseJson = """ - { - "error": { - "message": "You didn't provide an API key", - "type": "invalid_request_error", - "param": null, - "code": null - } - } - """; - - OpenAiErrorResponseEntity errorMessage = OpenAiErrorResponseEntity.fromResponse( - new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) - ); - assertEquals("You didn't provide an API key", errorMessage.getErrorMessage()); - } - - public void testFromResponse_noMessage() { - String responseJson = """ - { - "error": { - "type": "invalid_request_error" - } - } - """; - - OpenAiErrorResponseEntity errorMessage = OpenAiErrorResponseEntity.fromResponse( - new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) - ); - assertNull(errorMessage); - } - - public void testFromResponse_noError() { - String responseJson = """ - { - "something": { - "not": "relevant" - } - } - """; - - OpenAiErrorResponseEntity errorMessage = OpenAiErrorResponseEntity.fromResponse( - new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) - ); - assertNull(errorMessage); - } -} From b85e6491a91d11548b39540e87a3dcf536186d60 Mon Sep 17 00:00:00 2001 From: Patrick Doyle <810052+prdoyle@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:29:54 -0500 Subject: [PATCH 03/24] Relax assertion - remove unnecessary times(1) (#118497) --- muted-tests.yml | 3 --- .../service/FileSettingsServiceTests.java | 11 +++++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 613d3a3655ccf..50ad8c27675b4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -314,9 +314,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118220 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT issue: https://github.com/elastic/elasticsearch/issues/118238 -- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests - method: testInvalidJSON - issue: https://github.com/elastic/elasticsearch/issues/116521 # Examples: # diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index c19cf7c31bc68..b1cf40d7f22ec 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.junit.After; import org.junit.Before; +import org.mockito.Mockito; import org.mockito.stubbing.Answer; import java.io.IOException; @@ -315,7 +316,13 @@ public void testInvalidJSON() throws Exception { writeTestFile(fileSettingsService.watchedFile(), "test_invalid_JSON"); awaitOrBust(fileChangeBarrier); - verify(fileSettingsService, times(1)).onProcessFileChangesException( + // These checks use atLeast(1) because the initial JSON is also invalid, + // and so we sometimes get two calls to these error-reporting methods + // depending on timing. Rather than trace down the root cause and fix + // it, we tolerate this for now because, hey, invalid JSON is invalid JSON + // and this is still testing what we want to test. + + verify(fileSettingsService, Mockito.atLeast(1)).onProcessFileChangesException( argThat(e -> unwrapException(e) instanceof XContentParseException) ); @@ -324,7 +331,7 @@ public void testInvalidJSON() throws Exception { // of the watcher thread itself, which occurs asynchronously when clusterChanged is first called. assertEquals(YELLOW, healthIndicatorService.calculate(false, null).status()); - verify(healthIndicatorService).failureOccurred(contains(XContentParseException.class.getName())); + verify(healthIndicatorService, Mockito.atLeast(1)).failureOccurred(contains(XContentParseException.class.getName())); } /** From c6b42bb5026d1641703a3eb7f34fae331f663785 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:46:39 +1100 Subject: [PATCH 04/24] Mute org.elasticsearch.packaging.test.DockerTests test011SecurityEnabledStatus #118517 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 50ad8c27675b4..573bd035b872b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -314,6 +314,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/118220 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT issue: https://github.com/elastic/elasticsearch/issues/118238 +- class: org.elasticsearch.packaging.test.DockerTests + method: test011SecurityEnabledStatus + issue: https://github.com/elastic/elasticsearch/issues/118517 # Examples: # From a5607829958f15d92eddbfab59c2309ca0b0f5ea Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 11 Dec 2024 17:43:17 -0800 Subject: [PATCH 05/24] Remove any references to V_7 (#118103) This PR removes any references to org.elasticsearch.core.RestApiVersion#V_7. --- docs/changelog/118103.yaml | 11 +++++++++++ .../org/elasticsearch/core/RestApiVersion.java | 10 +--------- .../action/ingest/SimulatePipelineRequest.java | 2 +- .../common/xcontent/ChunkedToXContent.java | 1 - .../org/elasticsearch/ingest/IngestDocument.java | 3 +++ .../action/bulk/BulkRequestParserTests.java | 2 +- .../SimulatePipelineRequestParsingTests.java | 2 +- .../java/org/elasticsearch/license/License.java | 3 +-- .../xpack/core/ml/MachineLearningField.java | 3 --- .../xpack/core/ml/action/CloseJobAction.java | 13 +------------ .../core/ml/action/GetDatafeedsStatsAction.java | 2 ++ .../core/ml/action/GetOverallBucketsAction.java | 13 +------------ .../xpack/core/ml/action/StopDatafeedAction.java | 13 +------------ .../xpack/ml/rest/cat/RestCatDatafeedsAction.java | 14 +------------- .../xpack/ml/rest/cat/RestCatJobsAction.java | 14 +------------- .../rest/datafeeds/RestGetDatafeedStatsAction.java | 14 +------------- .../ml/rest/datafeeds/RestGetDatafeedsAction.java | 14 +------------- .../ml/rest/datafeeds/RestStopDatafeedAction.java | 14 +------------- .../xpack/ml/rest/job/RestCloseJobAction.java | 14 +------------- .../xpack/ml/rest/job/RestGetJobStatsAction.java | 14 +------------- .../xpack/ml/rest/job/RestGetJobsAction.java | 14 +------------- .../rest/results/RestGetOverallBucketsAction.java | 14 +------------- 22 files changed, 33 insertions(+), 171 deletions(-) create mode 100644 docs/changelog/118103.yaml diff --git a/docs/changelog/118103.yaml b/docs/changelog/118103.yaml new file mode 100644 index 0000000000000..c0bb8f56d6931 --- /dev/null +++ b/docs/changelog/118103.yaml @@ -0,0 +1,11 @@ +pr: 118103 +summary: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7" +area: Infra/Core +type: breaking +issues: [] +breaking: + title: "Remove any references to org.elasticsearch.core.RestApiVersion#V_7" + area: REST API + details: "This PR removes all references to V_7 in the Rest API. V7 features marked for deprecation have been removed." + impact: "This change is breaking for any external plugins/clients that rely on the V_7 enum or deprecated version 7 functionality" + notable: false diff --git a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java index 672090d195c5a..085887fdcdb0c 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java @@ -20,10 +20,7 @@ public enum RestApiVersion { V_9(9), - V_8(8), - - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // remove all references to V_7 then delete this annotation - V_7(7); + V_8(8); public final byte major; @@ -54,7 +51,6 @@ public static Predicate equalTo(RestApiVersion restApiVersion) { return switch (restApiVersion) { case V_9 -> r -> r.major == V_9.major; case V_8 -> r -> r.major == V_8.major; - case V_7 -> r -> r.major == V_7.major; }; } @@ -62,15 +58,11 @@ public static Predicate onOrAfter(RestApiVersion restApiVersion) return switch (restApiVersion) { case V_9 -> r -> r.major >= V_9.major; case V_8 -> r -> r.major >= V_8.major; - case V_7 -> r -> r.major >= V_7.major; }; } public static RestApiVersion forMajor(int major) { switch (major) { - case 7 -> { - return V_7; - } case 8 -> { return V_8; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index d6a2d81fdb7d3..02027b1f633d2 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -177,7 +177,7 @@ private static List parseDocs(Map config, RestAp String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.INDEX.getFieldName(), "_index"); String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, Metadata.ID.getFieldName(), "_id"); String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, dataMap, Metadata.ROUTING.getFieldName()); - if (restApiVersion != RestApiVersion.V_8 && dataMap.containsKey(Metadata.TYPE.getFieldName())) { + if (dataMap.containsKey(Metadata.TYPE.getFieldName())) { deprecationLogger.compatibleCritical( "simulate_pipeline_with_types", "[types removal] specifying _type in pipeline simulation requests is deprecated" diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java index db0b5b4357c7d..1970983654b88 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java @@ -49,7 +49,6 @@ static ChunkedToXContentBuilder builder(ToXContent.Params params) { */ default Iterator toXContentChunked(RestApiVersion restApiVersion, ToXContent.Params params) { return switch (restApiVersion) { - case V_7 -> throw new AssertionError("v7 API not supported"); case V_8 -> toXContentChunkedV8(params); case V_9 -> toXContentChunked(params); }; diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 280c7684a8553..0614e9e92edf2 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -957,6 +958,8 @@ void resetTerminate() { terminate = false; } + // Unconditionally deprecate the _type field once V7 BWC support is removed + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) public enum Metadata { INDEX(IndexFieldMapper.NAME), TYPE("_type"), diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java index 5785d076693e7..9d944d43f4c36 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestParserTests.java @@ -30,7 +30,7 @@ public class BulkRequestParserTests extends ESTestCase { @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) // Replace with just RestApiVersion.values() when V8 no longer exists public static final List REST_API_VERSIONS_POST_V8 = Stream.of(RestApiVersion.values()) - .filter(v -> v.compareTo(RestApiVersion.V_8) > 0) + .filter(v -> v.matches(RestApiVersion.onOrAfter(RestApiVersion.V_9))) .toList(); public void testParserCannotBeReusedAfterFailure() { diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index b8be05fbfb72d..391c258b6f098 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -342,7 +342,7 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception { requestContent, false, ingestService, - RestApiVersion.V_7 + RestApiVersion.V_8 ); assertThat(actualRequest.verbose(), equalTo(false)); assertThat(actualRequest.documents().size(), equalTo(numDocs)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java index f280dcf9b3edf..19790e61b6102 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xcontent.ToXContentObject; @@ -142,7 +141,7 @@ static boolean isEnterprise(String typeName) { */ public static final String LICENSE_VERSION_MODE = "license_version"; /** - * Set for {@link RestApiVersion#V_7} requests only + * Set for RestApiVersion#V_7 requests only * XContent param name to map the "enterprise" license type to "platinum" * for backwards compatibility with older clients */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java index 6c49cadb8d189..3a37f94e6b2d4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java @@ -22,9 +22,6 @@ public final class MachineLearningField { - public static final String DEPRECATED_ALLOW_NO_JOBS_PARAM = "allow_no_jobs"; - public static final String DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM = "allow_no_datafeeds"; - public static final Setting AUTODETECT_PROCESS = Setting.boolSetting( "xpack.ml.autodetect_process", true, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java index bddae0417e467..5963cff9746b6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java @@ -12,9 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.Task; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -27,10 +25,6 @@ import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.core.RestApiVersion.equalTo; -import static org.elasticsearch.core.RestApiVersion.onOrAfter; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; - public class CloseJobAction extends ActionType { public static final CloseJobAction INSTANCE = new CloseJobAction(); @@ -45,11 +39,7 @@ public static class Request extends BaseTasksRequest implements ToXCont public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField FORCE = new ParseField("force"); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8)); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 - public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_JOBS_PARAM) - .forRestApiVersion(equalTo(RestApiVersion.V_7)); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); public static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); static { @@ -60,7 +50,6 @@ public static class Request extends BaseTasksRequest implements ToXCont ); PARSER.declareBoolean(Request::setForce, FORCE); PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH); - PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH_V7); } public static Request parseRequest(String jobId, XContentParser parser) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java index fafb9afa99f85..eabd6ef5939e3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -62,6 +63,7 @@ private GetDatafeedsStatsAction() { // serialized to older nodes where the transport action was a MasterNodeReadAction. // TODO: Make this a simple request in a future version where there is no possibility // of this request being serialized to another node. + @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) public static class Request extends MasterNodeReadRequest { public static final String ALLOW_NO_MATCH = "allow_no_match"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java index 47bc6df5f6536..f9e91194fe31b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java @@ -13,9 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.time.DateMathParser; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -33,10 +31,6 @@ import java.util.Objects; import java.util.function.LongSupplier; -import static org.elasticsearch.core.RestApiVersion.equalTo; -import static org.elasticsearch.core.RestApiVersion.onOrAfter; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; - /** *

* This action returns summarized bucket results over multiple jobs. @@ -68,11 +62,7 @@ public static class Request extends ActionRequest implements ToXContentObject { public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim"); public static final ParseField START = new ParseField("start"); public static final ParseField END = new ParseField("end"); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8)); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 - public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_JOBS_PARAM) - .forRestApiVersion(equalTo(RestApiVersion.V_7)); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); @@ -88,7 +78,6 @@ public static class Request extends ActionRequest implements ToXContentObject { ); PARSER.declareString((request, endTime) -> request.setEnd(parseDateOrThrow(endTime, END, System::currentTimeMillis)), END); PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH); - PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH_V7); } static long parseDateOrThrow(String date, ParseField paramName, LongSupplier now) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java index bd4aac7ccad89..1278d1a57ec55 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java @@ -13,9 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.Task; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -29,10 +27,6 @@ import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.core.RestApiVersion.equalTo; -import static org.elasticsearch.core.RestApiVersion.onOrAfter; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM; - public class StopDatafeedAction extends ActionType { public static final StopDatafeedAction INSTANCE = new StopDatafeedAction(); @@ -47,11 +41,7 @@ public static class Request extends BaseTasksRequest implements ToXCont public static final ParseField TIMEOUT = new ParseField("timeout"); public static final ParseField FORCE = new ParseField("force"); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate forRestApiVersion - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match").forRestApiVersion(onOrAfter(RestApiVersion.V_8)); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 - public static final ParseField ALLOW_NO_MATCH_V7 = new ParseField("allow_no_match", DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM) - .forRestApiVersion(equalTo(RestApiVersion.V_7)); + public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); public static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); static { @@ -62,7 +52,6 @@ public static class Request extends BaseTasksRequest implements ToXCont ); PARSER.declareBoolean(Request::setForce, FORCE); PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH); - PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH_V7); } public static Request parseRequest(String datafeedId, XContentParser parser) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java index 205bb4f68a62c..417e9bca0a497 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java @@ -10,9 +10,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.Scope; @@ -30,8 +28,6 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestCatDatafeedsAction extends AbstractCatAction { @@ -52,16 +48,8 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c if (Strings.isNullOrEmpty(datafeedId)) { datafeedId = GetDatafeedsStatsAction.ALL; } - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request = new Request(datafeedId); - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM, - Request.ALLOW_NO_MATCH, - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH, request.allowNoMatch())); return channel -> client.execute(GetDatafeedsStatsAction.INSTANCE, request, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(Response getDatafeedsStatsRespons) throws Exception { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java index b27819bceee44..d8088c9510b6a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java @@ -12,9 +12,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.Scope; @@ -35,8 +33,6 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestCatJobsAction extends AbstractCatAction { @@ -57,16 +53,8 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c if (Strings.isNullOrEmpty(jobId)) { jobId = Metadata.ALL; } - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request = new Request(jobId); - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_JOBS_PARAM, - Request.ALLOW_NO_MATCH, - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH, request.allowNoMatch())); return channel -> client.execute(GetJobsStatsAction.INSTANCE, request, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(Response getJobStatsResponse) throws Exception { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java index 8c85c055fca3b..a4879eca46d39 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java @@ -8,8 +8,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -24,10 +22,8 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM; import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestGetDatafeedStatsAction extends BaseRestHandler { @@ -48,16 +44,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (Strings.isNullOrEmpty(datafeedId)) { datafeedId = GetDatafeedsStatsAction.ALL; } - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request = new Request(datafeedId); - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM, - Request.ALLOW_NO_MATCH, - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH, request.allowNoMatch())); return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( GetDatafeedsStatsAction.INSTANCE, request, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index fd0681f68a3a5..6955b81fdbb4c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.ml.rest.datafeeds; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -26,11 +24,9 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM; import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID; import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestGetDatafeedsAction extends BaseRestHandler { @@ -51,16 +47,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (datafeedId == null) { datafeedId = GetDatafeedsAction.ALL; } - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request = new Request(datafeedId); - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM, - GetDatafeedsStatsAction.Request.ALLOW_NO_MATCH, - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(GetDatafeedsStatsAction.Request.ALLOW_NO_MATCH, request.allowNoMatch())); return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( GetDatafeedsAction.INSTANCE, request, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java index 8235e2785cc37..dcc213a571469 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java @@ -7,9 +7,7 @@ package org.elasticsearch.xpack.ml.rest.datafeeds; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -28,10 +26,8 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM; import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig.ID; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestStopDatafeedAction extends BaseRestHandler { @@ -49,7 +45,6 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request; if (restRequest.hasContentOrSourceParam()) { XContentParser parser = restRequest.contentOrSourceParamParser(); @@ -63,14 +58,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (restRequest.hasParam(Request.FORCE.getPreferredName())) { request.setForce(restRequest.paramAsBoolean(Request.FORCE.getPreferredName(), request.isForce())); } - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM, - Request.ALLOW_NO_MATCH.getPreferredName(), - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH.getPreferredName(), request.allowNoMatch())); } return channel -> client.execute(StopDatafeedAction.INSTANCE, request, new RestBuilderListener(channel) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java index f98a2f5a933ae..56afb7d65dfe3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java @@ -7,9 +7,7 @@ package org.elasticsearch.xpack.ml.rest.job; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -23,10 +21,8 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; import static org.elasticsearch.xpack.core.ml.job.config.Job.ID; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestCloseJobAction extends BaseRestHandler { @@ -43,7 +39,6 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request; if (restRequest.hasContentOrSourceParam()) { request = Request.parseRequest(restRequest.param(Job.ID.getPreferredName()), restRequest.contentParser()); @@ -57,14 +52,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (restRequest.hasParam(Request.FORCE.getPreferredName())) { request.setForce(restRequest.paramAsBoolean(Request.FORCE.getPreferredName(), request.isForce())); } - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_JOBS_PARAM, - Request.ALLOW_NO_MATCH.getPreferredName(), - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH.getPreferredName(), request.allowNoMatch())); } return channel -> client.execute(CloseJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java index 2899faabdc40f..79951d7fad621 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java @@ -9,8 +9,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -25,10 +23,8 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; import static org.elasticsearch.xpack.core.ml.job.config.Job.ID; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestGetJobStatsAction extends BaseRestHandler { @@ -52,16 +48,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (Strings.isNullOrEmpty(jobId)) { jobId = Metadata.ALL; } - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request = new Request(jobId); - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_JOBS_PARAM, - Request.ALLOW_NO_MATCH, - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH, request.allowNoMatch())); return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( GetJobsStatsAction.INSTANCE, request, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index ae8d234d1d8bd..ea63a38ab01a8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -9,8 +9,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -27,11 +25,9 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; import static org.elasticsearch.xpack.core.ml.job.config.Job.ID; import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestGetJobsAction extends BaseRestHandler { @@ -52,16 +48,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (Strings.isNullOrEmpty(jobId)) { jobId = Metadata.ALL; } - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 Request request = new Request(jobId); - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_JOBS_PARAM, - Request.ALLOW_NO_MATCH, - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH, request.allowNoMatch())); return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute( GetJobsAction.INSTANCE, request, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java index 2700e01cb9f6b..c74f49efe0aed 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetOverallBucketsAction.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.ml.rest.results; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -24,10 +22,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.xpack.core.ml.MachineLearningField.DEPRECATED_ALLOW_NO_JOBS_PARAM; import static org.elasticsearch.xpack.core.ml.job.config.Job.ID; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; -import static org.elasticsearch.xpack.ml.rest.RestCompatibilityChecker.checkAndSetDeprecatedParam; @ServerlessScope(Scope.PUBLIC) public class RestGetOverallBucketsAction extends BaseRestHandler { @@ -48,7 +44,6 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String jobId = restRequest.param(Job.ID.getPreferredName()); - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // v7 REST API no longer exists: eliminate ref to RestApiVersion.V_7 final Request request; if (restRequest.hasContentOrSourceParam()) { XContentParser parser = restRequest.contentOrSourceParamParser(); @@ -67,14 +62,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient if (restRequest.hasParam(Request.END.getPreferredName())) { request.setEnd(restRequest.param(Request.END.getPreferredName())); } - checkAndSetDeprecatedParam( - DEPRECATED_ALLOW_NO_JOBS_PARAM, - Request.ALLOW_NO_MATCH.getPreferredName(), - RestApiVersion.V_7, - restRequest, - (r, s) -> r.paramAsBoolean(s, request.allowNoMatch()), - request::setAllowNoMatch - ); + request.setAllowNoMatch(restRequest.paramAsBoolean(Request.ALLOW_NO_MATCH.getPreferredName(), request.allowNoMatch())); } return channel -> client.execute(GetOverallBucketsAction.INSTANCE, request, new RestToXContentListener<>(channel)); From 7573312f2ad955c0af47b365860b6aed8c705460 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Thu, 12 Dec 2024 07:50:18 +0100 Subject: [PATCH 06/24] `_score` should not be a reserved attribute in ES|QL (#118435) --- docs/changelog/118435.yaml | 6 ++++ muted-tests.yml | 6 ++++ .../src/main/resources/scoring.csv-spec | 30 +++++++++++++++++++ .../xpack/esql/analysis/Verifier.java | 9 ------ .../xpack/esql/analysis/VerifierTests.java | 25 ---------------- 5 files changed, 42 insertions(+), 34 deletions(-) create mode 100644 docs/changelog/118435.yaml diff --git a/docs/changelog/118435.yaml b/docs/changelog/118435.yaml new file mode 100644 index 0000000000000..8bccbeb54698d --- /dev/null +++ b/docs/changelog/118435.yaml @@ -0,0 +1,6 @@ +pr: 118435 +summary: '`_score` should not be a reserved attribute in ES|QL' +area: ES|QL +type: enhancement +issues: + - 118460 diff --git a/muted-tests.yml b/muted-tests.yml index 573bd035b872b..5c99a2d5a5efd 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -180,6 +180,12 @@ tests: - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/117641 +- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT" + method: "test {scoring.*}" + issue: https://github.com/elastic/elasticsearch/issues/118460 +- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT" + method: "test {scoring.*}" + issue: https://github.com/elastic/elasticsearch/issues/118460 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT method: test {scoring.QstrWithFieldAndScoringSortedEval} issue: https://github.com/elastic/elasticsearch/issues/117751 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index d4c7b8c59fdbc..cb38204a71ab0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -283,3 +283,33 @@ book_no:keyword | c_score:double 7350 | 2.0 7140 | 3.0 ; + +QstrScoreManipulation +required_capability: metadata_score +required_capability: qstr_function + +from books metadata _score +| where qstr("title:rings") +| eval _score = _score + 1 +| keep book_no, title, _score +| limit 2; + +book_no:keyword | title:text | _score:double +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 2.6404519081115723 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 2.9239964485168457 +; + +QstrScoreOverride +required_capability: metadata_score +required_capability: qstr_function + +from books metadata _score +| where qstr("title:rings") +| eval _score = "foobar" +| keep book_no, title, _score +| limit 2; + +book_no:keyword | title:text | _score:keyword +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | foobar +2714 | Return of the King Being the Third Part of The Lord of the Rings | foobar +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index a0728c9a91088..c805adf5d5a57 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; @@ -208,7 +207,6 @@ else if (p instanceof Lookup lookup) { checkJoin(p, failures); }); checkRemoteEnrich(plan, failures); - checkMetadataScoreNameReserved(plan, failures); if (failures.isEmpty()) { checkLicense(plan, licenseState, failures); @@ -222,13 +220,6 @@ else if (p instanceof Lookup lookup) { return failures; } - private static void checkMetadataScoreNameReserved(LogicalPlan p, Set failures) { - // _score can only be set as metadata attribute - if (p.inputSet().stream().anyMatch(a -> MetadataAttribute.SCORE.equals(a.name()) && (a instanceof MetadataAttribute) == false)) { - failures.add(fail(p, "`" + MetadataAttribute.SCORE + "` is a reserved METADATA attribute")); - } - } - private void checkSort(LogicalPlan p, Set failures) { if (p instanceof OrderBy ob) { ob.order().forEach(o -> { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index d58d233168e2b..84dcdbadef9f0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; -import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; @@ -22,7 +21,6 @@ import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.parser.QueryParams; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -1805,29 +1803,6 @@ public void testToDatePeriodToTimeDurationWithInvalidType() { ); } - public void testNonMetadataScore() { - assumeTrue("'METADATA _score' is disabled", EsqlCapabilities.Cap.METADATA_SCORE.isEnabled()); - assertEquals("1:12: `_score` is a reserved METADATA attribute", error("from foo | eval _score = 10")); - - assertEquals( - "1:48: `_score` is a reserved METADATA attribute", - error("from foo metadata _score | where qstr(\"bar\") | eval _score = _score + 1") - ); - } - - public void testScoreRenaming() { - assumeTrue("'METADATA _score' is disabled", EsqlCapabilities.Cap.METADATA_SCORE.isEnabled()); - assertEquals("1:33: `_score` is a reserved METADATA attribute", error("from foo METADATA _id, _score | rename _id as _score")); - - assertTrue(passes("from foo metadata _score | rename _score as foo").stream().anyMatch(a -> a.name().equals("foo"))); - } - - private List passes(String query) { - LogicalPlan logicalPlan = defaultAnalyzer.analyze(parser.createStatement(query)); - assertTrue(logicalPlan.resolved()); - return logicalPlan.output(); - } - public void testIntervalAsString() { // DateTrunc for (String interval : List.of("1 minu", "1 dy", "1.5 minutes", "0.5 days", "minutes 1", "day 5")) { From ff8e5e964e545984063dc21b28e6c8da0f5acbcb Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Thu, 12 Dec 2024 09:41:14 +0200 Subject: [PATCH 07/24] Improve InputStreamIndexInput testSkipBytes (#118485) Relates ES-10234 --- .../store/InputStreamIndexInputTests.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java b/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java index 4bea6f50c7c4b..b982bd7b95aad 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterIndexInput; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -267,17 +268,47 @@ public void testSkipBytes() throws Exception { skipBytesExpected ); - IndexInput input = dir.openInput("test", IOContext.DEFAULT); - InputStreamIndexInput is = new InputStreamIndexInput(input, limit); + var countingInput = new CountingReadBytesIndexInput("test", dir.openInput("test", IOContext.DEFAULT)); + InputStreamIndexInput is = new InputStreamIndexInput(countingInput, limit); is.readNBytes(initialReadBytes); assertThat(is.skip(skipBytes), equalTo((long) skipBytesExpected)); + long expectedActualInitialBytesRead = Math.min(Math.min(initialReadBytes, limit), bytes); + assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead)); int remainingBytes = Math.min(bytes, limit) - seekExpected; for (int i = seekExpected; i < seekExpected + remainingBytes; i++) { assertThat(is.read(), equalTo(i)); } + assertThat(countingInput.getBytesRead(), equalTo(expectedActualInitialBytesRead + remainingBytes)); } + protected static class CountingReadBytesIndexInput extends FilterIndexInput { + private long bytesRead = 0; + + public CountingReadBytesIndexInput(String resourceDescription, IndexInput in) { + super(resourceDescription, in); + } + + @Override + public byte readByte() throws IOException { + long filePointerBefore = getFilePointer(); + byte b = super.readByte(); + bytesRead += getFilePointer() - filePointerBefore; + return b; + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + long filePointerBefore = getFilePointer(); + super.readBytes(b, offset, len); + bytesRead += getFilePointer() - filePointerBefore; + } + + public long getBytesRead() { + return bytesRead; + } + }; + public void testReadZeroShouldReturnZero() throws IOException { try (Directory dir = new ByteBuffersDirectory()) { try (IndexOutput output = dir.createOutput("test", IOContext.DEFAULT)) { From 80a1a6f7afef990a7243a2e415866c6746113e28 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 12 Dec 2024 08:59:52 +0100 Subject: [PATCH 08/24] ESQL: Add CCS tests for FLS and DLS against data streams (#118423) CCS test coverage for https://github.com/elastic/elasticsearch/pull/118378 --- .../security/qa/multi-cluster/build.gradle | 1 + ...teClusterSecurityDataStreamEsqlRcs1IT.java | 402 ++++++++++++++++++ ...teClusterSecurityDataStreamEsqlRcs2IT.java | 126 ++++++ .../src/javaRestTest/resources/roles.yml | 99 +++++ 4 files changed, 628 insertions(+) create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle index 5b682cfdccade..f4eee4ef46c02 100644 --- a/x-pack/plugin/security/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle @@ -24,6 +24,7 @@ dependencies { clusterModules project(':x-pack:plugin:enrich') clusterModules project(':x-pack:plugin:autoscaling') clusterModules project(':x-pack:plugin:ml') + clusterModules project(xpackModule('ilm')) clusterModules(project(":modules:ingest-common")) } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java new file mode 100644 index 0000000000000..57eb583912c49 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java @@ -0,0 +1,402 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.Build; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.Strings; +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; + +// TODO consolidate me with RemoteClusterSecurityDataStreamEsqlRcs2IT +public class RemoteClusterSecurityDataStreamEsqlRcs1IT extends AbstractRemoteClusterSecurityTestCase { + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.authc.token.enabled", "true") + .rolesFile(Resource.fromClasspath("roles.yml")) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.authc.token.enabled", "true") + .rolesFile(Resource.fromClasspath("roles.yml")) + .user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false) + .user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false) + .user("logs_foo_after_2021", "x-pack-test-password", "logs_foo_after_2021", false) + .user("logs_foo_after_2021_pattern", "x-pack-test-password", "logs_foo_after_2021_pattern", false) + .user("logs_foo_after_2021_alias", "x-pack-test-password", "logs_foo_after_2021_alias", false) + .build(); + } + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + public void testDataStreamsWithDlsAndFls() throws Exception { + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), randomBoolean()); + createDataStreamOnFulfillingCluster(); + setupAdditionalUsersAndRoles(); + + doTestDataStreamsWithFlsAndDls(); + } + + private void setupAdditionalUsersAndRoles() throws IOException { + createUserAndRoleOnQueryCluster("fls_user_logs_pattern", "fls_user_logs_pattern", """ + { + "indices": [ + { + "names": ["logs-*"], + "privileges": ["read"], + "field_security": { + "grant": ["@timestamp", "data_stream.namespace"] + } + } + ] + }"""); + createUserAndRoleOnFulfillingCluster("fls_user_logs_pattern", "fls_user_logs_pattern", """ + { + "indices": [ + { + "names": ["logs-*"], + "privileges": ["read"], + "field_security": { + "grant": ["@timestamp", "data_stream.namespace"] + } + } + ] + }"""); + } + + static void createUserAndRoleOnQueryCluster(String username, String roleName, String roleJson) throws IOException { + final var putRoleRequest = new Request("PUT", "/_security/role/" + roleName); + putRoleRequest.setJsonEntity(roleJson); + assertOK(adminClient().performRequest(putRoleRequest)); + + final var putUserRequest = new Request("PUT", "/_security/user/" + username); + putUserRequest.setJsonEntity(Strings.format(""" + { + "password": "%s", + "roles" : ["%s"] + }""", PASS, roleName)); + assertOK(adminClient().performRequest(putUserRequest)); + } + + static void createUserAndRoleOnFulfillingCluster(String username, String roleName, String roleJson) throws IOException { + final var putRoleRequest = new Request("PUT", "/_security/role/" + roleName); + putRoleRequest.setJsonEntity(roleJson); + assertOK(performRequestAgainstFulfillingCluster(putRoleRequest)); + + final var putUserRequest = new Request("PUT", "/_security/user/" + username); + putUserRequest.setJsonEntity(Strings.format(""" + { + "password": "%s", + "roles" : ["%s"] + }""", PASS, roleName)); + assertOK(performRequestAgainstFulfillingCluster(putUserRequest)); + } + + static Response runESQLCommandAgainstQueryCluster(String user, String command) throws IOException { + if (command.toLowerCase(Locale.ROOT).contains("limit") == false) { + // add a (high) limit to avoid warnings on default limit + command += " | limit 10000000"; + } + XContentBuilder json = JsonXContent.contentBuilder(); + json.startObject(); + json.field("query", command); + addRandomPragmas(json); + json.endObject(); + Request request = new Request("POST", "_query"); + request.setJsonEntity(org.elasticsearch.common.Strings.toString(json)); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user)); + request.addParameter("error_trace", "true"); + Response response = adminClient().performRequest(request); + assertOK(response); + return response; + } + + static void addRandomPragmas(XContentBuilder builder) throws IOException { + if (Build.current().isSnapshot()) { + Settings pragmas = randomPragmas(); + if (pragmas != Settings.EMPTY) { + builder.startObject("pragma"); + builder.value(pragmas); + builder.endObject(); + } + } + } + + static Settings randomPragmas() { + Settings.Builder settings = Settings.builder(); + if (randomBoolean()) { + settings.put("page_size", between(1, 5)); + } + if (randomBoolean()) { + settings.put("exchange_buffer_size", between(1, 2)); + } + if (randomBoolean()) { + settings.put("data_partitioning", randomFrom("shard", "segment", "doc")); + } + if (randomBoolean()) { + settings.put("enrich_max_workers", between(1, 5)); + } + if (randomBoolean()) { + settings.put("node_level_reduction", randomBoolean()); + } + return settings.build(); + } + + static void createDataStreamOnFulfillingCluster() throws Exception { + createDataStreamPolicy(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamComponentTemplate(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamIndexTemplate(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamDocuments(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamAlias(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + } + + private static void createDataStreamPolicy(CheckedFunction requestConsumer) throws Exception { + Request request = new Request("PUT", "_ilm/policy/my-lifecycle-policy"); + request.setJsonEntity(""" + { + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "50gb" + } + } + }, + "delete": { + "min_age": "735d", + "actions": { + "delete": {} + } + } + } + } + }"""); + + requestConsumer.apply(request); + } + + private static void createDataStreamComponentTemplate(CheckedFunction requestConsumer) throws Exception { + Request request = new Request("PUT", "_component_template/my-template"); + request.setJsonEntity(""" + { + "template": { + "settings": { + "index.lifecycle.name": "my-lifecycle-policy" + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "data_stream": { + "properties": { + "namespace": {"type": "keyword"}, + "environment": {"type": "keyword"} + } + } + } + } + } + }"""); + requestConsumer.apply(request); + } + + private static void createDataStreamIndexTemplate(CheckedFunction requestConsumer) throws Exception { + Request request = new Request("PUT", "_index_template/my-index-template"); + request.setJsonEntity(""" + { + "index_patterns": ["logs-*"], + "data_stream": {}, + "composed_of": ["my-template"], + "priority": 500 + }"""); + requestConsumer.apply(request); + } + + private static void createDataStreamDocuments(CheckedFunction requestConsumer) throws Exception { + Request request = new Request("POST", "logs-foo/_bulk"); + request.addParameter("refresh", ""); + request.setJsonEntity(""" + { "create" : {} } + { "@timestamp": "2099-05-06T16:21:15.000Z", "data_stream": {"namespace": "16", "environment": "dev"} } + { "create" : {} } + { "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17", "environment": "prod"} } + """); + assertMap(entityAsMap(requestConsumer.apply(request)), matchesMap().extraOk().entry("errors", false)); + } + + private static void createDataStreamAlias(CheckedFunction requestConsumer) throws Exception { + Request request = new Request("PUT", "_alias"); + request.setJsonEntity(""" + { + "actions": [ + { + "add": { + "index": "logs-foo", + "alias": "alias-foo" + } + } + ] + }"""); + assertMap(entityAsMap(requestConsumer.apply(request)), matchesMap().extraOk().entry("errors", false)); + } + + static void doTestDataStreamsWithFlsAndDls() throws IOException { + // DLS + MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); + MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), + oneResult + ); + + // FLS + // logs_foo_all does not have FLS restrictions so should be able to access all fields + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | SORT data_stream.namespace | LIMIT 1") + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.environment").entry("type", "keyword"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | SORT data_stream.namespace | LIMIT 1") + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.environment").entry("type", "keyword"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster( + "fls_user_logs_pattern", + "FROM my_remote_cluster:logs-foo | SORT data_stream.namespace | LIMIT 1" + ) + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster( + "fls_user_logs_pattern", + "FROM my_remote_cluster:logs-* | SORT data_stream.namespace | LIMIT 1" + ) + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java new file mode 100644 index 0000000000000..c5cf704177020 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.createDataStreamOnFulfillingCluster; +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.createUserAndRoleOnQueryCluster; +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.doTestDataStreamsWithFlsAndDls; + +// TODO consolidate me with RemoteClusterSecurityDataStreamEsqlRcs1IT +public class RemoteClusterSecurityDataStreamEsqlRcs2IT extends AbstractRemoteClusterSecurityTestCase { + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(3) + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get()))) + .node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get()))) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["logs-*", "alias-*"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false) + .user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false) + .user("logs_foo_after_2021", "x-pack-test-password", "logs_foo_after_2021", false) + .user("logs_foo_after_2021_pattern", "x-pack-test-password", "logs_foo_after_2021_pattern", false) + .user("logs_foo_after_2021_alias", "x-pack-test-password", "logs_foo_after_2021_alias", false) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + // `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters + // We set it here, since randomization methods are not available in the static initialize context above + public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> { + SSL_ENABLED_REF.set(usually()); + NODE1_RCS_SERVER_ENABLED.set(randomBoolean()); + NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); + })).around(fulfillingCluster).around(queryCluster); + + public void testDataStreamsWithDlsAndFls() throws Exception { + configureRemoteCluster(); + createDataStreamOnFulfillingCluster(); + setupAdditionalUsersAndRoles(); + + doTestDataStreamsWithFlsAndDls(); + } + + private void setupAdditionalUsersAndRoles() throws IOException { + createUserAndRoleOnQueryCluster("fls_user_logs_pattern", "fls_user_logs_pattern", """ + { + "indices": [{"names": [""], "privileges": ["read"]}], + "remote_indices": [ + { + "names": ["logs-*"], + "privileges": ["read"], + "field_security": { + "grant": ["@timestamp", "data_stream.namespace"] + }, + "clusters": ["*"] + } + ] + }"""); + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml index b61daa068ed1a..c09f9dc620a4c 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml @@ -41,3 +41,102 @@ ccr_user_role: manage_role: cluster: [ 'manage' ] + +logs_foo_all: + cluster: [] + indices: + - names: [ 'logs-foo' ] + privileges: [ 'read' ] + remote_indices: + - names: [ 'logs-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + +logs_foo_16_only: + cluster: [] + indices: + - names: [ 'logs-foo' ] + privileges: [ 'read' ] + query: | + { + "term": { + "data_stream.namespace": "16" + } + } + remote_indices: + - names: [ 'logs-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "term": { + "data_stream.namespace": "16" + } + } + +logs_foo_after_2021: + cluster: [] + indices: + - names: [ 'logs-foo' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + remote_indices: + - names: [ 'logs-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + +logs_foo_after_2021_pattern: + cluster: [] + indices: + - names: [ 'logs-*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + remote_indices: + - names: [ 'logs-*' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + +logs_foo_after_2021_alias: + cluster: [] + indices: + - names: [ 'alias-foo' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + remote_indices: + - names: [ 'alias-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + From 95315cc08c702cbb53c0cf6544773be58bbc373f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 12 Dec 2024 09:28:58 +0100 Subject: [PATCH 09/24] Building scope -> entitlements map during PolicyManager initialization (#118070) --- .../runtime/policy/PolicyManager.java | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index a77c86d5ffd04..8d3efe4eb98e6 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -18,7 +18,6 @@ import java.lang.module.ModuleFinder; import java.lang.module.ModuleReference; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.IdentityHashMap; import java.util.List; @@ -56,8 +55,8 @@ public Stream getEntitlements(Class entitlementCla final Map moduleEntitlementsMap = new HashMap<>(); - protected final Policy serverPolicy; - protected final Map pluginPolicies; + protected final Map> serverEntitlements; + protected final Map>> pluginsEntitlements; private final Function, String> pluginResolver; public static final String ALL_UNNAMED = "ALL-UNNAMED"; @@ -79,19 +78,16 @@ private static Set findSystemModules() { } public PolicyManager(Policy defaultPolicy, Map pluginPolicies, Function, String> pluginResolver) { - this.serverPolicy = Objects.requireNonNull(defaultPolicy); - this.pluginPolicies = Collections.unmodifiableMap(Objects.requireNonNull(pluginPolicies)); + this.serverEntitlements = buildScopeEntitlementsMap(Objects.requireNonNull(defaultPolicy)); + this.pluginsEntitlements = Objects.requireNonNull(pluginPolicies) + .entrySet() + .stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue()))); this.pluginResolver = pluginResolver; } - private static List lookupEntitlementsForModule(Policy policy, String moduleName) { - for (int i = 0; i < policy.scopes.size(); ++i) { - var scope = policy.scopes.get(i); - if (scope.name.equals(moduleName)) { - return scope.entitlements; - } - } - return null; + private static Map> buildScopeEntitlementsMap(Policy policy) { + return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements)); } public void checkExitVM(Class callerClass) { @@ -141,21 +137,21 @@ ModuleEntitlements getEntitlementsOrThrow(Class callerClass, Module requestin if (isServerModule(requestingModule)) { var scopeName = requestingModule.getName(); - return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverPolicy, scopeName); + return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName); } // plugins var pluginName = pluginResolver.apply(callerClass); if (pluginName != null) { - var pluginPolicy = pluginPolicies.get(pluginName); - if (pluginPolicy != null) { + var pluginEntitlements = pluginsEntitlements.get(pluginName); + if (pluginEntitlements != null) { final String scopeName; if (requestingModule.isNamed() == false) { scopeName = ALL_UNNAMED; } else { scopeName = requestingModule.getName(); } - return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginPolicy, scopeName); + return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName); } } @@ -167,15 +163,20 @@ private static String buildModuleNoPolicyMessage(Class callerClass, Module re return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName()); } - private ModuleEntitlements getModuleEntitlementsOrThrow(Class callerClass, Module module, Policy policy, String moduleName) { - var entitlements = lookupEntitlementsForModule(policy, moduleName); + private ModuleEntitlements getModuleEntitlementsOrThrow( + Class callerClass, + Module module, + Map> scopeEntitlements, + String moduleName + ) { + var entitlements = scopeEntitlements.get(moduleName); if (entitlements == null) { // Module without entitlements - remember we don't have any moduleEntitlementsMap.put(module, ModuleEntitlements.NONE); throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module)); } // We have a policy for this module - var classEntitlements = createClassEntitlements(entitlements); + var classEntitlements = new ModuleEntitlements(entitlements); moduleEntitlementsMap.put(module, classEntitlements); return classEntitlements; } @@ -184,10 +185,6 @@ private static boolean isServerModule(Module requestingModule) { return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot(); } - private ModuleEntitlements createClassEntitlements(List entitlements) { - return new ModuleEntitlements(entitlements); - } - private static Module requestingModule(Class callerClass) { if (callerClass != null) { Module callerModule = callerClass.getModule(); @@ -222,6 +219,6 @@ private static boolean isTriviallyAllowed(Module requestingModule) { @Override public String toString() { - return "PolicyManager{" + "serverPolicy=" + serverPolicy + ", pluginPolicies=" + pluginPolicies + '}'; + return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}'; } } From adddfa27ad69d270139e05b9e65123740f9225a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 12 Dec 2024 09:36:50 +0100 Subject: [PATCH 10/24] Add one test for plugin type to `PluginsLoaderTests` (#117725) * Add one test for plugin type to PluginsLoaderTests * Suppress ExtraFs (or PluginsUtils etc could fail with extra0 files) --- .../elasticsearch/plugins/PluginsLoader.java | 6 +- .../plugins/PluginsLoaderTests.java | 249 ++++++++++++++++++ 2 files changed, 252 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java index aadda93f977b6..8dfc1fc27c6aa 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java @@ -416,7 +416,7 @@ static String toModuleName(String name) { return result; } - static final String toPackageName(String className) { + static String toPackageName(String className) { assert className.endsWith(".") == false; int index = className.lastIndexOf('.'); if (index == -1) { @@ -426,11 +426,11 @@ static final String toPackageName(String className) { } @SuppressForbidden(reason = "I need to convert URL's to Paths") - static final Path[] urlsToPaths(Set urls) { + static Path[] urlsToPaths(Set urls) { return urls.stream().map(PluginsLoader::uncheckedToURI).map(PathUtils::get).toArray(Path[]::new); } - static final URI uncheckedToURI(URL url) { + static URI uncheckedToURI(URL url) { try { return url.toURI(); } catch (URISyntaxException e) { diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java index 059cb15551acb..b7d63b7d612c9 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsLoaderTests.java @@ -9,12 +9,45 @@ package org.elasticsearch.plugins; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.plugin.analysis.CharFilterFactory; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.PrivilegedOperations; +import org.elasticsearch.test.compiler.InMemoryJavaCompiler; +import org.elasticsearch.test.jar.JarUtils; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static java.util.Map.entry; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +@ESTestCase.WithoutSecurityManager +@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") public class PluginsLoaderTests extends ESTestCase { + private static final Logger logger = LogManager.getLogger(PluginsLoaderTests.class); + + static PluginsLoader newPluginsLoader(Settings settings) { + return PluginsLoader.createPluginsLoader(null, TestEnvironment.newEnvironment(settings).pluginsFile(), false); + } + public void testToModuleName() { assertThat(PluginsLoader.toModuleName("module.name"), equalTo("module.name")); assertThat(PluginsLoader.toModuleName("module-name"), equalTo("module.name")); @@ -28,4 +61,220 @@ public void testToModuleName() { assertThat(PluginsLoader.toModuleName("_module_name"), equalTo("_module_name")); assertThat(PluginsLoader.toModuleName("_"), equalTo("_")); } + + public void testStablePluginLoading() throws Exception { + final Path home = createTempDir(); + final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build(); + final Path plugins = home.resolve("plugins"); + final Path plugin = plugins.resolve("stable-plugin"); + Files.createDirectories(plugin); + PluginTestUtil.writeStablePluginProperties( + plugin, + "description", + "description", + "name", + "stable-plugin", + "version", + "1.0.0", + "elasticsearch.version", + Version.CURRENT.toString(), + "java.version", + System.getProperty("java.specification.version") + ); + + Path jar = plugin.resolve("impl.jar"); + JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """ + package p; + import java.util.Map; + import org.elasticsearch.plugin.analysis.CharFilterFactory; + import org.elasticsearch.plugin.NamedComponent; + import java.io.Reader; + @NamedComponent( "a_name") + public class A implements CharFilterFactory { + @Override + public Reader create(Reader reader) { + return reader; + } + } + """))); + Path namedComponentFile = plugin.resolve("named_components.json"); + Files.writeString(namedComponentFile, """ + { + "org.elasticsearch.plugin.analysis.CharFilterFactory": { + "a_name": "p.A" + } + } + """); + + var pluginsLoader = newPluginsLoader(settings); + try { + var loadedLayers = pluginsLoader.pluginLayers().toList(); + + assertThat(loadedLayers, hasSize(1)); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().getName(), equalTo("stable-plugin")); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().isStable(), is(true)); + + assertThat(pluginsLoader.pluginDescriptors(), hasSize(1)); + assertThat(pluginsLoader.pluginDescriptors().get(0).getName(), equalTo("stable-plugin")); + assertThat(pluginsLoader.pluginDescriptors().get(0).isStable(), is(true)); + + var pluginClassLoader = loadedLayers.get(0).pluginClassLoader(); + var pluginModuleLayer = loadedLayers.get(0).pluginModuleLayer(); + assertThat(pluginClassLoader, instanceOf(UberModuleClassLoader.class)); + assertThat(pluginModuleLayer, is(not(ModuleLayer.boot()))); + assertThat(pluginModuleLayer.modules(), contains(transformedMatch(Module::getName, equalTo("synthetic.stable.plugin")))); + + if (CharFilterFactory.class.getModule().isNamed() == false) { + // test frameworks run with stable api classes on classpath, so we + // have no choice but to let our class read the unnamed module that + // owns the stable api classes + ((UberModuleClassLoader) pluginClassLoader).addReadsSystemClassLoaderUnnamedModule(); + } + + Class stableClass = pluginClassLoader.loadClass("p.A"); + assertThat(stableClass.getModule().getName(), equalTo("synthetic.stable.plugin")); + } finally { + closePluginLoaders(pluginsLoader); + } + } + + public void testModularPluginLoading() throws Exception { + final Path home = createTempDir(); + final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build(); + final Path plugins = home.resolve("plugins"); + final Path plugin = plugins.resolve("modular-plugin"); + Files.createDirectories(plugin); + PluginTestUtil.writePluginProperties( + plugin, + "description", + "description", + "name", + "modular-plugin", + "classname", + "p.A", + "modulename", + "modular.plugin", + "version", + "1.0.0", + "elasticsearch.version", + Version.CURRENT.toString(), + "java.version", + System.getProperty("java.specification.version") + ); + + Path jar = plugin.resolve("impl.jar"); + Map sources = Map.ofEntries(entry("module-info", "module modular.plugin { exports p; }"), entry("p.A", """ + package p; + import org.elasticsearch.plugins.Plugin; + + public class A extends Plugin { + } + """)); + + // Usually org.elasticsearch.plugins.Plugin would be in the org.elasticsearch.server module. + // Unfortunately, as tests run non-modular, it will be in the unnamed module, so we need to add a read for it. + var classToBytes = InMemoryJavaCompiler.compile(sources, "--add-reads", "modular.plugin=ALL-UNNAMED"); + + JarUtils.createJarWithEntries( + jar, + Map.ofEntries(entry("module-info.class", classToBytes.get("module-info")), entry("p/A.class", classToBytes.get("p.A"))) + ); + + var pluginsLoader = newPluginsLoader(settings); + try { + var loadedLayers = pluginsLoader.pluginLayers().toList(); + + assertThat(loadedLayers, hasSize(1)); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().getName(), equalTo("modular-plugin")); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().isStable(), is(false)); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().isModular(), is(true)); + + assertThat(pluginsLoader.pluginDescriptors(), hasSize(1)); + assertThat(pluginsLoader.pluginDescriptors().get(0).getName(), equalTo("modular-plugin")); + assertThat(pluginsLoader.pluginDescriptors().get(0).isModular(), is(true)); + + var pluginModuleLayer = loadedLayers.get(0).pluginModuleLayer(); + assertThat(pluginModuleLayer, is(not(ModuleLayer.boot()))); + assertThat(pluginModuleLayer.modules(), contains(transformedMatch(Module::getName, equalTo("modular.plugin")))); + } finally { + closePluginLoaders(pluginsLoader); + } + } + + public void testNonModularPluginLoading() throws Exception { + final Path home = createTempDir(); + final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build(); + final Path plugins = home.resolve("plugins"); + final Path plugin = plugins.resolve("non-modular-plugin"); + Files.createDirectories(plugin); + PluginTestUtil.writePluginProperties( + plugin, + "description", + "description", + "name", + "non-modular-plugin", + "classname", + "p.A", + "version", + "1.0.0", + "elasticsearch.version", + Version.CURRENT.toString(), + "java.version", + System.getProperty("java.specification.version") + ); + + Path jar = plugin.resolve("impl.jar"); + Map sources = Map.ofEntries(entry("p.A", """ + package p; + import org.elasticsearch.plugins.Plugin; + + public class A extends Plugin { + } + """)); + + var classToBytes = InMemoryJavaCompiler.compile(sources); + + JarUtils.createJarWithEntries(jar, Map.ofEntries(entry("p/A.class", classToBytes.get("p.A")))); + + var pluginsLoader = newPluginsLoader(settings); + try { + var loadedLayers = pluginsLoader.pluginLayers().toList(); + + assertThat(loadedLayers, hasSize(1)); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().getName(), equalTo("non-modular-plugin")); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().isStable(), is(false)); + assertThat(loadedLayers.get(0).pluginBundle().pluginDescriptor().isModular(), is(false)); + + assertThat(pluginsLoader.pluginDescriptors(), hasSize(1)); + assertThat(pluginsLoader.pluginDescriptors().get(0).getName(), equalTo("non-modular-plugin")); + assertThat(pluginsLoader.pluginDescriptors().get(0).isModular(), is(false)); + + var pluginModuleLayer = loadedLayers.get(0).pluginModuleLayer(); + assertThat(pluginModuleLayer, is(ModuleLayer.boot())); + } finally { + closePluginLoaders(pluginsLoader); + } + } + + // Closes the URLClassLoaders and UberModuleClassloaders created by the given plugin loader. + // We can use the direct ClassLoader from the plugin because tests do not use any parent SPI ClassLoaders. + static void closePluginLoaders(PluginsLoader pluginsLoader) { + pluginsLoader.pluginLayers().forEach(lp -> { + if (lp.pluginClassLoader() instanceof URLClassLoader urlClassLoader) { + try { + PrivilegedOperations.closeURLClassLoader(urlClassLoader); + } catch (IOException unexpected) { + throw new UncheckedIOException(unexpected); + } + } else if (lp.pluginClassLoader() instanceof UberModuleClassLoader loader) { + try { + PrivilegedOperations.closeURLClassLoader(loader.getInternalLoader()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + logger.info("Cannot close unexpected classloader " + lp.pluginClassLoader()); + } + }); + } } From c30ba12e6b679c31fc89a5a58d912b1c40f9ad52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:59:10 +0100 Subject: [PATCH 11/24] Disable check_on_startup for KibanaUserRoleIntegTests (#118428) --- muted-tests.yml | 6 ------ .../integration/KibanaUserRoleIntegTests.java | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 5c99a2d5a5efd..c91c7b50a0808 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -52,12 +52,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item} issue: https://github.com/elastic/elasticsearch/issues/113325 -- class: org.elasticsearch.integration.KibanaUserRoleIntegTests - method: testFieldMappings - issue: https://github.com/elastic/elasticsearch/issues/113592 -- class: org.elasticsearch.integration.KibanaUserRoleIntegTests - method: testSearchAndMSearch - issue: https://github.com/elastic/elasticsearch/issues/113593 - class: org.elasticsearch.xpack.transform.integration.TransformIT method: testStopWaitForCheckpoint issue: https://github.com/elastic/elasticsearch/issues/106113 diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java index 7d99d5817bdc0..bc730b5695c19 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java @@ -14,13 +14,16 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import java.util.Map; +import java.util.Random; import static java.util.Collections.singletonMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; @@ -60,6 +63,14 @@ public String configUsersRoles() { return super.configUsersRoles() + "my_kibana_user:kibana_user\n" + "kibana_user:kibana_user"; } + @Override + protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) { + // Prevent INDEX_CHECK_ON_STARTUP as a random setting since it could result in indices being checked for corruption before opening. + // When corruption is detected, it will prevent the shard from being opened. This check is expensive in terms of CPU and memory + // usage and causes intermittent CI failures due to timeout. + return super.setRandomIndexSettings(random, builder).put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), false); + } + public void testFieldMappings() throws Exception { final String index = "logstash-20-12-2015"; final String field = "foo"; From 671c7d7c766bf3dfa58317c75587ca7d49b6985f Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 12 Dec 2024 10:40:05 +0100 Subject: [PATCH 12/24] Adjust OldCodecAvailableTests for v9 (#118510) N-2 indices won't be covered by archive indices, but rather supported out-of-the-box. This commit reflects that in the tests expectations and adjusts the related update for v9 annotation. --- .../xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java index 0b72b96b446d4..18cf1b49f5f37 100644 --- a/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java +++ b/x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.lucene.bwc.codecs; import org.elasticsearch.Version; -import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.test.ESTestCase; public class OldCodecsAvailableTests extends ESTestCase { @@ -17,10 +17,8 @@ public class OldCodecsAvailableTests extends ESTestCase { * Reminder to add Lucene BWC codecs under {@link org.elasticsearch.xpack.lucene.bwc.codecs} whenever Elasticsearch is upgraded * to the next major Lucene version. */ - @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) - @AwaitsFix(bugUrl = "muted until we add bwc codecs to support 7.x indices in Elasticsearch 9.0") + @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS) public void testLuceneBWCCodecsAvailable() { - assertEquals("Add Lucene BWC codecs for Elasticsearch version 7", 8, Version.CURRENT.major); + assertEquals("Add Lucene BWC codecs for Elasticsearch version 7", 9, Version.CURRENT.major); } - } From 8fb6edab636c462563dc2e7bfc1eab7f1ea8c4b5 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:52:41 +0100 Subject: [PATCH 13/24] [DOCS] Consolidate connectors release notes on one page (#118464) --- .../docs/connectors-release-notes.asciidoc | 74 ++++++++++++++++++- .../connectors-release-notes-8.16.0.asciidoc | 53 ------------- 2 files changed, 71 insertions(+), 56 deletions(-) delete mode 100644 docs/reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc diff --git a/docs/reference/connector/docs/connectors-release-notes.asciidoc b/docs/reference/connector/docs/connectors-release-notes.asciidoc index e1ed082365c00..ff3d859e1a888 100644 --- a/docs/reference/connector/docs/connectors-release-notes.asciidoc +++ b/docs/reference/connector/docs/connectors-release-notes.asciidoc @@ -9,8 +9,76 @@ Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation]. ==== -*Release notes*: +[discrete] +[[es-connectors-release-notes-8-17-0]] +=== 8.17.0 -* <> +No notable changes in this release. -include::release-notes/connectors-release-notes-8.16.0.asciidoc[] +[discrete] +[[es-connectors-release-notes-8-16-1]] +=== 8.16.1 + +[discrete] +[[es-connectors-release-notes-8-16-1-bug-fixes]] +==== Bug fixes + +* Fixed a bug in the Outlook Connector where having deactivated users could cause the sync to fail. +See https://github.com/elastic/connectors/pull/2967[*PR 2967*]. +* Fixed a bug where the Confluence connector was not downloading some blog post documents due to unexpected response format. +See https://github.com/elastic/connectors/pull/2984[*PR 2984*]. + +[discrete] +[[es-connectors-release-notes-8-16-0]] +=== 8.16.0 + +[discrete] +[[es-connectors-release-notes-deprecation-notice]] +==== Deprecation notices + +* *Direct index access for connectors and sync jobs* ++ +IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release. + +* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <> by default. +See https://github.com/elastic/connectors/pull/2884[*PR 2902*]. + +* *Docker `enterprise-search` namespace deprecation* ++ +IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. ++ +Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service]. ++ +During this transition period, images are published to both namespaces: ++ +** *Example*: ++ +Deprecated namespace:: +`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0` ++ +New namespace:: +`docker.elastic.co/integrations/elastic-connectors:v8.16.0` ++ +Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases. + +[discrete] +[[es-connectors-release-notes-8-16-0-enhancements]] +==== Enhancements + +* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base. + +* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`. +See https://github.com/elastic/connectors/pull/2762[*PR 2762*]. + +* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+`, aligning with Elastic Agent and conforming to SEMVER. +See https://github.com/elastic/connectors/pull/2749[*PR 2749*]. + +* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output. +See https://github.com/elastic/connectors/pull/2695[*PR 2695*]. + +[discrete] +[[es-connectors-release-notes-8-16-0-bug-fixes]] +==== Bug fixes + +* The Dropbox connector now fetches the files from team shared folders. +See https://github.com/elastic/connectors/pull/2718[*PR 2718*]. diff --git a/docs/reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc b/docs/reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc deleted file mode 100644 index 7608336073176..0000000000000 --- a/docs/reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc +++ /dev/null @@ -1,53 +0,0 @@ -[[es-connectors-release-notes-8-16-0]] -=== 8.16.0 connectors release notes - -[discrete] -[[es-connectors-release-notes-deprecation-notice]] -==== Deprecation notices - -* *Direct index access for connectors and sync jobs* -+ -IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release. - -* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <> by default. -See https://github.com/elastic/connectors/pull/2884[*PR 2902*]. - -* *Docker `enterprise-search` namespace deprecation* -+ -IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. -+ -Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service]. -+ -During this transition period, images are published to both namespaces: -+ -** *Example*: -+ -Deprecated namespace:: -`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0` -+ -New namespace:: -`docker.elastic.co/integrations/elastic-connectors:v8.16.0` -+ -Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases. - -[discrete] -[[es-connectors-release-notes-8-16-0-enhancements]] -==== Enhancements - -* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base. - -* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`. -See https://github.com/elastic/connectors/pull/2762[*PR 2762*]. - -* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+`, aligning with Elastic Agent and conforming to SEMVER. -See https://github.com/elastic/connectors/pull/2749[*PR 2749*]. - -* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output. -See https://github.com/elastic/connectors/pull/2695[*PR 2695*]. - -[discrete] -[[es-connectors-release-notes-8-16-0-bug-fixes]] -==== Bug fixes - -* The Dropbox connector now fetches the files from team shared folders. -See https://github.com/elastic/connectors/pull/2718[*PR 2718*]. \ No newline at end of file From 8a4fc7ceca1b853be06cba812fcf3da9b29f9606 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 12 Dec 2024 10:02:01 +0000 Subject: [PATCH 14/24] Remove most uses of 7.5 and 7.6 transport versions (#118439) --- .../index/rankeval/RankEvalRequest.java | 9 +--- .../elasticsearch/ElasticsearchException.java | 9 +--- .../org/elasticsearch/TransportVersions.java | 1 - ...ransportNodesListGatewayStartedShards.java | 12 +---- .../transport/RemoteConnectionInfo.java | 48 ++++--------------- .../StringStatsAggregationBuilder.java | 2 +- .../xpack/core/ccr/AutoFollowMetadata.java | 10 +--- .../core/enrich/EnrichFeatureSetUsage.java | 2 +- .../xpack/core/enrich/EnrichMetadata.java | 2 +- .../security/SecurityFeatureSetUsage.java | 8 +--- .../SamlPrepareAuthenticationRequest.java | 9 +--- .../xpack/core/slm/SLMFeatureSetUsage.java | 2 +- .../transform/TransformFeatureSetUsage.java | 2 +- .../transforms/TransformStateTests.java | 38 --------------- 14 files changed, 27 insertions(+), 127 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java index 8e0d838e602e7..4bb30fdb0dd01 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -46,9 +45,7 @@ public RankEvalRequest(RankEvalSpec rankingEvaluationSpec, String[] indices) { rankingEvaluationSpec = new RankEvalSpec(in); indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) { - searchType = SearchType.fromId(in.readByte()); - } + searchType = SearchType.fromId(in.readByte()); } RankEvalRequest() {} @@ -127,9 +124,7 @@ public void writeTo(StreamOutput out) throws IOException { rankingEvaluationSpec.writeTo(out); out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) { - out.writeByte(searchType.id()); - } + out.writeByte(searchType.id()); } @Override diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 11736bfe07deb..a430611559bb4 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -40,7 +40,6 @@ import org.elasticsearch.persistent.PersistentTaskNodeNotAssignedException; import org.elasticsearch.rest.ApiNotAvailableException; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.SearchException; import org.elasticsearch.search.TooManyScrollContextsException; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; @@ -319,10 +318,6 @@ protected void writeTo(StreamOutput out, Writer nestedExceptionsWrite public static ElasticsearchException readException(StreamInput input, int id) throws IOException { CheckedFunction elasticsearchException = ID_TO_SUPPLIER.get(id); if (elasticsearchException == null) { - if (id == 127 && input.getTransportVersion().before(TransportVersions.V_7_5_0)) { - // was SearchContextException - return new SearchException(input); - } throw new IllegalStateException("unknown exception for id: " + id); } return elasticsearchException.apply(input); @@ -1817,13 +1812,13 @@ private enum ElasticsearchExceptionHandle { org.elasticsearch.index.seqno.RetentionLeaseInvalidRetainingSeqNoException.class, org.elasticsearch.index.seqno.RetentionLeaseInvalidRetainingSeqNoException::new, 156, - TransportVersions.V_7_5_0 + UNKNOWN_VERSION_ADDED ), INGEST_PROCESSOR_EXCEPTION( org.elasticsearch.ingest.IngestProcessorException.class, org.elasticsearch.ingest.IngestProcessorException::new, 157, - TransportVersions.V_7_5_0 + UNKNOWN_VERSION_ADDED ), PEER_RECOVERY_NOT_FOUND_EXCEPTION( org.elasticsearch.indices.recovery.PeerRecoveryNotFound.class, diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index d61afbdf98587..ac083862357d6 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -54,7 +54,6 @@ static TransportVersion def(int id) { public static final TransportVersion V_7_0_0 = def(7_00_00_99); public static final TransportVersion V_7_3_0 = def(7_03_00_99); public static final TransportVersion V_7_4_0 = def(7_04_00_99); - public static final TransportVersion V_7_5_0 = def(7_05_00_99); public static final TransportVersion V_7_6_0 = def(7_06_00_99); public static final TransportVersion V_7_7_0 = def(7_07_00_99); public static final TransportVersion V_7_8_0 = def(7_08_00_99); diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java index 737b904849835..c3c33d5de9a00 100644 --- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -239,11 +238,7 @@ public static class NodeRequest extends TransportRequest { public NodeRequest(StreamInput in) throws IOException { super(in); shardId = new ShardId(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) { - customDataPath = in.readString(); - } else { - customDataPath = null; - } + customDataPath = in.readString(); } public NodeRequest(Request request) { @@ -255,10 +250,7 @@ public NodeRequest(Request request) { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); shardId.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) { - assert customDataPath != null; - out.writeString(customDataPath); - } + out.writeString(customDataPath); } public ShardId getShardId() { diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java index 7a579cbb98bc2..ae078479ba85c 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java @@ -18,8 +18,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Arrays; -import java.util.List; import java.util.Objects; /** @@ -49,25 +47,14 @@ public RemoteConnectionInfo( } public RemoteConnectionInfo(StreamInput input) throws IOException { - if (input.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) { - RemoteConnectionStrategy.ConnectionStrategy mode = input.readEnum(RemoteConnectionStrategy.ConnectionStrategy.class); - modeInfo = mode.getReader().read(input); - initialConnectionTimeout = input.readTimeValue(); - clusterAlias = input.readString(); - skipUnavailable = input.readBoolean(); - if (input.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { - hasClusterCredentials = input.readBoolean(); - } else { - hasClusterCredentials = false; - } + RemoteConnectionStrategy.ConnectionStrategy mode = input.readEnum(RemoteConnectionStrategy.ConnectionStrategy.class); + modeInfo = mode.getReader().read(input); + initialConnectionTimeout = input.readTimeValue(); + clusterAlias = input.readString(); + skipUnavailable = input.readBoolean(); + if (input.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { + hasClusterCredentials = input.readBoolean(); } else { - List seedNodes = Arrays.asList(input.readStringArray()); - int connectionsPerCluster = input.readVInt(); - initialConnectionTimeout = input.readTimeValue(); - int numNodesConnected = input.readVInt(); - clusterAlias = input.readString(); - skipUnavailable = input.readBoolean(); - modeInfo = new SniffConnectionStrategy.SniffModeInfo(seedNodes, connectionsPerCluster, numNodesConnected); hasClusterCredentials = false; } } @@ -90,24 +77,9 @@ public boolean hasClusterCredentials() { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) { - out.writeEnum(modeInfo.modeType()); - modeInfo.writeTo(out); - out.writeTimeValue(initialConnectionTimeout); - } else { - if (modeInfo.modeType() == RemoteConnectionStrategy.ConnectionStrategy.SNIFF) { - SniffConnectionStrategy.SniffModeInfo sniffInfo = (SniffConnectionStrategy.SniffModeInfo) this.modeInfo; - out.writeStringCollection(sniffInfo.seedNodes); - out.writeVInt(sniffInfo.maxConnectionsPerCluster); - out.writeTimeValue(initialConnectionTimeout); - out.writeVInt(sniffInfo.numNodesConnected); - } else { - out.writeStringArray(new String[0]); - out.writeVInt(0); - out.writeTimeValue(initialConnectionTimeout); - out.writeVInt(0); - } - } + out.writeEnum(modeInfo.modeType()); + modeInfo.writeTo(out); + out.writeTimeValue(initialConnectionTimeout); out.writeString(clusterAlias); out.writeBoolean(skipUnavailable); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java index c75ed46102112..ea5a0adffa0eb 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java @@ -154,6 +154,6 @@ public boolean equals(Object obj) { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_6_0; + return TransportVersions.ZERO; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java index 6b54f7a7dddce..5276d7659fb02 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java @@ -282,11 +282,7 @@ private AutoFollowPattern( this.leaderIndexPatterns = leaderIndexPatterns; this.followIndexPattern = followIndexPattern; this.settings = Objects.requireNonNull(settings); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_5_0)) { - this.active = in.readBoolean(); - } else { - this.active = true; - } + this.active = in.readBoolean(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_14_0)) { this.leaderIndexExclusionPatterns = in.readStringCollectionAsList(); } else { @@ -351,9 +347,7 @@ public void writeTo(StreamOutput out) throws IOException { settings.writeTo(out); } super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_5_0)) { - out.writeBoolean(active); - } + out.writeBoolean(active); if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_14_0)) { out.writeStringCollection(leaderIndexExclusionPatterns); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java index 819b3d86b68c8..b51fa386ddeea 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java @@ -27,6 +27,6 @@ public EnrichFeatureSetUsage(StreamInput input) throws IOException { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_5_0; + return TransportVersions.ZERO; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java index b949e44ef036a..30ecaf2ff680f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java @@ -84,7 +84,7 @@ public EnumSet context() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_5_0; + return TransportVersions.ZERO; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java index 726797d2e563a..f44409daa37f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java @@ -61,9 +61,7 @@ public SecurityFeatureSetUsage(StreamInput in) throws IOException { ipFilterUsage = in.readGenericMap(); anonymousUsage = in.readGenericMap(); roleMappingStoreUsage = in.readGenericMap(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_5_0)) { - fips140Usage = in.readGenericMap(); - } + fips140Usage = in.readGenericMap(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_11_0)) { operatorPrivilegesUsage = in.readGenericMap(); } @@ -129,9 +127,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeGenericMap(ipFilterUsage); out.writeGenericMap(anonymousUsage); out.writeGenericMap(roleMappingStoreUsage); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_5_0)) { - out.writeGenericMap(fips140Usage); - } + out.writeGenericMap(fips140Usage); if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_11_0)) { out.writeGenericMap(operatorPrivilegesUsage); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java index 3b5ddb21be91c..067bb156aa909 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.security.action.saml; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.io.stream.StreamInput; @@ -33,9 +32,7 @@ public SamlPrepareAuthenticationRequest(StreamInput in) throws IOException { super(in); realmName = in.readOptionalString(); assertionConsumerServiceURL = in.readOptionalString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_5_0)) { - relayState = in.readOptionalString(); - } + relayState = in.readOptionalString(); } public SamlPrepareAuthenticationRequest() {} @@ -87,8 +84,6 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalString(realmName); out.writeOptionalString(assertionConsumerServiceURL); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_5_0)) { - out.writeOptionalString(relayState); - } + out.writeOptionalString(relayState); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java index 099eaa2468e1c..53b45827c4a9f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java @@ -41,7 +41,7 @@ public SLMFeatureSetUsage(@Nullable SnapshotLifecycleStats slmStats) { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_5_0; + return TransportVersions.ZERO; } public SnapshotLifecycleStats getStats() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java index e4c15a3b9007c..909bf6858eab0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java @@ -50,7 +50,7 @@ public TransformFeatureSetUsage( @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_5_0; + return TransportVersions.ZERO; } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java index e01549032be5e..0acf46edc4bdd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java @@ -7,9 +7,6 @@ package org.elasticsearch.xpack.core.transform.transforms; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -66,39 +63,4 @@ protected boolean supportsUnknownFields() { protected Predicate getRandomFieldsExcludeFilter() { return field -> field.isEmpty() == false; } - - public void testBackwardsSerialization() throws IOException { - TransformState state = new TransformState( - randomFrom(TransformTaskState.values()), - randomFrom(IndexerState.values()), - TransformIndexerPositionTests.randomTransformIndexerPosition(), - randomLongBetween(0, 10), - randomBoolean() ? null : randomAlphaOfLength(10), - randomBoolean() ? null : randomTransformProgress(), - randomBoolean() ? null : randomNodeAttributes(), - false, - randomBoolean() ? null : AuthorizationStateTests.randomAuthorizationState() - ); - // auth_state will be null after BWC deserialization - TransformState expectedState = new TransformState( - state.getTaskState(), - state.getIndexerState(), - state.getPosition(), - state.getCheckpoint(), - state.getReason(), - state.getProgress(), - state.getNode(), - state.shouldStopAtNextCheckpoint(), - null - ); - try (BytesStreamOutput output = new BytesStreamOutput()) { - output.setTransportVersion(TransportVersions.V_7_5_0); - state.writeTo(output); - try (StreamInput in = output.bytes().streamInput()) { - in.setTransportVersion(TransportVersions.V_7_5_0); - TransformState streamedState = new TransformState(in); - assertEquals(expectedState, streamedState); - } - } - } } From 8bbc6b314149163be3fa26d7f9066cc79f68a866 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Thu, 12 Dec 2024 12:05:45 +0200 Subject: [PATCH 15/24] Suppress the for-loop warnings since it is a conscious performance choice. (#118530) --- .../cluster/metadata/IndexNameExpressionResolver.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index e7914d812e05c..2ce91b66fa789 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -66,6 +66,7 @@ * Note: This class is performance sensitive, so we pay extra attention on the data structure usage and we avoid streams and iterators * when possible in favor of the classic for-i loops. */ +@SuppressWarnings("ForLoopReplaceableByForEach") public class IndexNameExpressionResolver { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexNameExpressionResolver.class); From 3cf7f9714189b514429948d45455d558dd0f4bf7 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:17:16 +0100 Subject: [PATCH 16/24] Adds CCS matrix for 8.17 (#118527) Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- .../ccs-version-compat-matrix.asciidoc | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc index 5859ccd03e511..a68f20fb1c656 100644 --- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc +++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc @@ -1,25 +1,26 @@ |==== -| 20+^h| Remote cluster version +| 21+^h| Remote cluster version h| Local cluster version - | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 | 8.15 | 8.16 -| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.15 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 8.16 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} + | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 | 8.15 | 8.16 | 8.17 +| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} +| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} +| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.15 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.16 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.17 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} |==== From a8484ad8efe78be8e6a95e8cc37ed5f96098c469 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 12 Dec 2024 10:21:05 +0000 Subject: [PATCH 17/24] [ML] Fix timeout ingesting an empty string into a semantic_text field (#117840) --- docs/changelog/117840.yaml | 5 ++ .../chunking/SentenceBoundaryChunker.java | 8 ++- .../chunking/WordBoundaryChunker.java | 4 -- .../EmbeddingRequestChunkerTests.java | 51 ++++++++++++++++++- .../SentenceBoundaryChunkerTests.java | 35 +++++++++++++ .../chunking/WordBoundaryChunkerTests.java | 34 +++++++++++-- .../xpack/ml/integration/PyTorchModelIT.java | 16 ++++++ .../TransportInternalInferModelAction.java | 5 ++ 8 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/117840.yaml diff --git a/docs/changelog/117840.yaml b/docs/changelog/117840.yaml new file mode 100644 index 0000000000000..e1f469643af42 --- /dev/null +++ b/docs/changelog/117840.yaml @@ -0,0 +1,5 @@ +pr: 117840 +summary: Fix timeout ingesting an empty string into a `semantic_text` field +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java index b2d6c83b89211..bf28e30074a9d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java @@ -62,7 +62,8 @@ public List chunk(String input, ChunkingSettings chunkingSettings) * * @param input Text to chunk * @param maxNumberWordsPerChunk Maximum size of the chunk - * @return The input text chunked + * @param includePrecedingSentence Include the previous sentence + * @return The input text offsets */ public List chunk(String input, int maxNumberWordsPerChunk, boolean includePrecedingSentence) { var chunks = new ArrayList(); @@ -158,6 +159,11 @@ public List chunk(String input, int maxNumberWordsPerChunk, boolean chunks.add(new ChunkOffset(chunkStart, input.length())); } + if (chunks.isEmpty()) { + // The input did not chunk, return the entire input + chunks.add(new ChunkOffset(0, input.length())); + } + return chunks; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java index b15e2134f4cf7..1ce90a9e416e5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunker.java @@ -96,10 +96,6 @@ List chunkPositions(String input, int chunkSize, int overlap) { throw new IllegalArgumentException("Invalid chunking parameters, overlap [" + overlap + "] must be >= 0"); } - if (input.isEmpty()) { - return List.of(); - } - var chunkPositions = new ArrayList(); // This position in the chunk is where the next overlapping chunk will start diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index a82d2f474ca4a..dec7d15760aa6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; +import org.hamcrest.Matchers; import java.util.ArrayList; import java.util.List; @@ -31,16 +32,62 @@ public class EmbeddingRequestChunkerTests extends ESTestCase { - public void testEmptyInput() { + public void testEmptyInput_WordChunker() { var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); var batches = new EmbeddingRequestChunker(List.of(), 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); assertThat(batches, empty()); } - public void testBlankInput() { + public void testEmptyInput_SentenceChunker() { + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); + var batches = new EmbeddingRequestChunker(List.of(), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) + .batchRequestsWithListeners(testListener()); + assertThat(batches, empty()); + } + + public void testWhitespaceInput_SentenceChunker() { + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); + var batches = new EmbeddingRequestChunker(List.of(" "), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) + .batchRequestsWithListeners(testListener()); + assertThat(batches, hasSize(1)); + assertThat(batches.get(0).batch().inputs(), hasSize(1)); + assertThat(batches.get(0).batch().inputs().get(0), Matchers.is(" ")); + } + + public void testBlankInput_WordChunker() { var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); var batches = new EmbeddingRequestChunker(List.of(""), 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); + assertThat(batches.get(0).batch().inputs(), hasSize(1)); + assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("")); + } + + public void testBlankInput_SentenceChunker() { + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); + var batches = new EmbeddingRequestChunker(List.of(""), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) + .batchRequestsWithListeners(testListener()); + assertThat(batches, hasSize(1)); + assertThat(batches.get(0).batch().inputs(), hasSize(1)); + assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("")); + } + + public void testInputThatDoesNotChunk_WordChunker() { + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); + var batches = new EmbeddingRequestChunker(List.of("ABBAABBA"), 100, 100, 10, embeddingType).batchRequestsWithListeners( + testListener() + ); + assertThat(batches, hasSize(1)); + assertThat(batches.get(0).batch().inputs(), hasSize(1)); + assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("ABBAABBA")); + } + + public void testInputThatDoesNotChunk_SentenceChunker() { + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); + var batches = new EmbeddingRequestChunker(List.of("ABBAABBA"), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) + .batchRequestsWithListeners(testListener()); + assertThat(batches, hasSize(1)); + assertThat(batches.get(0).batch().inputs(), hasSize(1)); + assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("ABBAABBA")); } public void testShortInputsAreSingleBatch() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java index de943f7f57ab8..f81894ccd4bbb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java @@ -43,6 +43,41 @@ private List textChunks( return chunkPositions.stream().map(offset -> input.substring(offset.start(), offset.end())).collect(Collectors.toList()); } + public void testEmptyString() { + var chunks = textChunks(new SentenceBoundaryChunker(), "", 100, randomBoolean()); + assertThat(chunks, hasSize(1)); + assertThat(chunks.get(0), Matchers.is("")); + } + + public void testBlankString() { + var chunks = textChunks(new SentenceBoundaryChunker(), " ", 100, randomBoolean()); + assertThat(chunks, hasSize(1)); + assertThat(chunks.get(0), Matchers.is(" ")); + } + + public void testSingleChar() { + var chunks = textChunks(new SentenceBoundaryChunker(), " b", 100, randomBoolean()); + assertThat(chunks, Matchers.contains(" b")); + + chunks = textChunks(new SentenceBoundaryChunker(), "b", 100, randomBoolean()); + assertThat(chunks, Matchers.contains("b")); + + chunks = textChunks(new SentenceBoundaryChunker(), ". ", 100, randomBoolean()); + assertThat(chunks, Matchers.contains(". ")); + + chunks = textChunks(new SentenceBoundaryChunker(), " , ", 100, randomBoolean()); + assertThat(chunks, Matchers.contains(" , ")); + + chunks = textChunks(new SentenceBoundaryChunker(), " ,", 100, randomBoolean()); + assertThat(chunks, Matchers.contains(" ,")); + } + + public void testSingleCharRepeated() { + var input = "a".repeat(32_000); + var chunks = textChunks(new SentenceBoundaryChunker(), input, 100, randomBoolean()); + assertThat(chunks, Matchers.contains(input)); + } + public void testChunkSplitLargeChunkSizes() { for (int maxWordsPerChunk : new int[] { 100, 200 }) { var chunker = new SentenceBoundaryChunker(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java index 2ef28f2cf2e77..b4fa5c9122258 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; import java.util.List; import java.util.Locale; @@ -71,10 +72,6 @@ public class WordBoundaryChunkerTests extends ESTestCase { * Use the chunk functions that return offsets where possible */ List textChunks(WordBoundaryChunker chunker, String input, int chunkSize, int overlap) { - if (input.isEmpty()) { - return List.of(""); - } - var chunkPositions = chunker.chunk(input, chunkSize, overlap); return chunkPositions.stream().map(p -> input.substring(p.start(), p.end())).collect(Collectors.toList()); } @@ -240,6 +237,35 @@ public void testWhitespace() { assertThat(chunks, contains(" ")); } + public void testBlankString() { + var chunks = textChunks(new WordBoundaryChunker(), " ", 100, 10); + assertThat(chunks, hasSize(1)); + assertThat(chunks.get(0), Matchers.is(" ")); + } + + public void testSingleChar() { + var chunks = textChunks(new WordBoundaryChunker(), " b", 100, 10); + assertThat(chunks, Matchers.contains(" b")); + + chunks = textChunks(new WordBoundaryChunker(), "b", 100, 10); + assertThat(chunks, Matchers.contains("b")); + + chunks = textChunks(new WordBoundaryChunker(), ". ", 100, 10); + assertThat(chunks, Matchers.contains(". ")); + + chunks = textChunks(new WordBoundaryChunker(), " , ", 100, 10); + assertThat(chunks, Matchers.contains(" , ")); + + chunks = textChunks(new WordBoundaryChunker(), " ,", 100, 10); + assertThat(chunks, Matchers.contains(" ,")); + } + + public void testSingleCharRepeated() { + var input = "a".repeat(32_000); + var chunks = textChunks(new WordBoundaryChunker(), input, 100, 10); + assertThat(chunks, Matchers.contains(input)); + } + public void testPunctuation() { int chunkSize = 1; var chunks = textChunks(new WordBoundaryChunker(), "Comma, separated", chunkSize, 0); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java index 4e92cad1026a3..04f349d67d7fe 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java @@ -1142,6 +1142,22 @@ public void testDeploymentThreadsIncludedInUsage() throws IOException { } } + public void testInferEmptyInput() throws IOException { + String modelId = "empty_input"; + createPassThroughModel(modelId); + putModelDefinition(modelId); + putVocabulary(List.of("these", "are", "my", "words"), modelId); + startDeployment(modelId); + + Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=30s"); + request.setJsonEntity(""" + { "docs": [] } + """); + + var inferenceResponse = client().performRequest(request); + assertThat(EntityUtils.toString(inferenceResponse.getEntity()), equalTo("{\"inference_results\":[]}")); + } + private void putModelDefinition(String modelId) throws IOException { putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index e0405b1749536..20a4ceeae59b3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -132,6 +132,11 @@ protected void doExecute(Task task, Request request, ActionListener li Response.Builder responseBuilder = Response.builder(); TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId()); + if (request.numberOfDocuments() == 0) { + listener.onResponse(responseBuilder.setId(request.getId()).build()); + return; + } + if (MachineLearning.INFERENCE_AGG_FEATURE.check(licenseState)) { responseBuilder.setLicensed(true); doInfer(task, request, responseBuilder, parentTaskId, listener); From a975927320fc5660b67388fbe636f8b332ad4ef5 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 12 Dec 2024 10:57:57 +0000 Subject: [PATCH 18/24] Add `discovery-ec2` integration test for AZ attr (#118452) Verifies that the plugin sets the `aws_availability_zone` automatically by reading the AZ name from the IMDS at startup. --- .../s3/RepositoryS3EcsCredentialsRestIT.java | 6 +- .../RepositoryS3ImdsV1CredentialsRestIT.java | 7 +-- .../RepositoryS3ImdsV2CredentialsRestIT.java | 7 +-- plugins/discovery-ec2/build.gradle | 3 + ...yEc2AvailabilityZoneAttributeImdsV1IT.java | 37 ++++++++++++ ...yEc2AvailabilityZoneAttributeImdsV2IT.java | 37 ++++++++++++ ...yEc2AvailabilityZoneAttributeNoImdsIT.java | 37 ++++++++++++ ...yEc2AvailabilityZoneAttributeTestCase.java | 52 +++++++++++++++++ .../fixture/aws/imds/Ec2ImdsHttpFixture.java | 24 ++------ .../fixture/aws/imds/Ec2ImdsHttpHandler.java | 13 ++++- .../aws/imds/Ec2ImdsServiceBuilder.java | 57 +++++++++++++++++++ .../aws/imds/Ec2ImdsHttpHandlerTests.java | 26 ++++++--- 12 files changed, 265 insertions(+), 41 deletions(-) create mode 100644 plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java create mode 100644 plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java create mode 100644 plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java create mode 100644 plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java create mode 100644 test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java index a79ae4de7cc66..4f0bf83000642 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.repositories.s3; import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; import fixture.aws.imds.Ec2ImdsVersion; import fixture.s3.DynamicS3Credentials; import fixture.s3.S3HttpFixture; @@ -37,9 +38,8 @@ public class RepositoryS3EcsCredentialsRestIT extends AbstractRepositoryS3RestTe private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials(); private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( - Ec2ImdsVersion.V1, - dynamicS3Credentials::addValidCredentials, - Set.of("/ecs_credentials_endpoint") + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials) + .alternativeCredentialsEndpoints(Set.of("/ecs_credentials_endpoint")) ); private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized); diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java index ead91981b3fa8..dcdf52e963eef 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.repositories.s3; import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; import fixture.aws.imds.Ec2ImdsVersion; import fixture.s3.DynamicS3Credentials; import fixture.s3.S3HttpFixture; @@ -23,8 +24,6 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import java.util.Set; - @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3RestTestCase { @@ -37,9 +36,7 @@ public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3Res private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials(); private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( - Ec2ImdsVersion.V1, - dynamicS3Credentials::addValidCredentials, - Set.of() + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials) ); private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized); diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java index 67adb096bd1ba..434fc9720fc29 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.repositories.s3; import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; import fixture.aws.imds.Ec2ImdsVersion; import fixture.s3.DynamicS3Credentials; import fixture.s3.S3HttpFixture; @@ -23,8 +24,6 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import java.util.Set; - @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3RestTestCase { @@ -37,9 +36,7 @@ public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3Res private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials(); private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( - Ec2ImdsVersion.V2, - dynamicS3Credentials::addValidCredentials, - Set.of() + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials) ); private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized); diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 2335577225340..d4b56015edaa8 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -26,6 +26,9 @@ dependencies { api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" api "joda-time:joda-time:2.10.10" + + javaRestTestImplementation project(':plugins:discovery-ec2') + javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture') } tasks.named("dependencyLicenses").configure { diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java new file mode 100644 index 0000000000000..32291236ea158 --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +public class DiscoveryEc2AvailabilityZoneAttributeImdsV1IT extends DiscoveryEc2AvailabilityZoneAttributeTestCase { + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).availabilityZoneSupplier( + DiscoveryEc2AvailabilityZoneAttributeTestCase::getAvailabilityZone + ) + ); + + public static ElasticsearchCluster cluster = buildCluster(ec2ImdsHttpFixture::getAddress); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java new file mode 100644 index 0000000000000..8b785d688e7c4 --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import fixture.aws.imds.Ec2ImdsHttpFixture; +import fixture.aws.imds.Ec2ImdsServiceBuilder; +import fixture.aws.imds.Ec2ImdsVersion; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +public class DiscoveryEc2AvailabilityZoneAttributeImdsV2IT extends DiscoveryEc2AvailabilityZoneAttributeTestCase { + private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture( + new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).availabilityZoneSupplier( + DiscoveryEc2AvailabilityZoneAttributeTestCase::getAvailabilityZone + ) + ); + + public static ElasticsearchCluster cluster = buildCluster(ec2ImdsHttpFixture::getAddress); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(cluster); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java new file mode 100644 index 0000000000000..602a98e17970d --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import org.elasticsearch.client.Request; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +import java.io.IOException; + +public class DiscoveryEc2AvailabilityZoneAttributeNoImdsIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .plugin("discovery-ec2") + .setting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testAvailabilityZoneAttribute() throws IOException { + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none"))); + for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { + assertNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone")); + } + } +} diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java new file mode 100644 index 0000000000000..7eb18eec5c0b9 --- /dev/null +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.discovery.ec2; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +public abstract class DiscoveryEc2AvailabilityZoneAttributeTestCase extends ESRestTestCase { + + private static final Set createdAvailabilityZones = ConcurrentCollections.newConcurrentSet(); + + protected static String getAvailabilityZone() { + final var zoneName = randomIdentifier(); + createdAvailabilityZones.add(zoneName); + return zoneName; + } + + protected static ElasticsearchCluster buildCluster(Supplier imdsFixtureAddressSupplier) { + return ElasticsearchCluster.local() + .plugin("discovery-ec2") + .setting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), "true") + .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", imdsFixtureAddressSupplier) + .build(); + } + + public void testAvailabilityZoneAttribute() throws IOException { + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none"))); + for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { + assertThat( + createdAvailabilityZones, + Matchers.hasItem( + Objects.requireNonNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone")) + ) + ); + } + } +} diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java index c63c65a750d7c..cc268a6021cb3 100644 --- a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java @@ -8,7 +8,6 @@ */ package fixture.aws.imds; -import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; import org.junit.rules.ExternalResource; @@ -17,29 +16,14 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.Objects; -import java.util.Set; -import java.util.function.BiConsumer; public class Ec2ImdsHttpFixture extends ExternalResource { + private final Ec2ImdsServiceBuilder ec2ImdsServiceBuilder; private HttpServer server; - private final Ec2ImdsVersion ec2ImdsVersion; - private final BiConsumer newCredentialsConsumer; - private final Set alternativeCredentialsEndpoints; - - public Ec2ImdsHttpFixture( - Ec2ImdsVersion ec2ImdsVersion, - BiConsumer newCredentialsConsumer, - Set alternativeCredentialsEndpoints - ) { - this.ec2ImdsVersion = Objects.requireNonNull(ec2ImdsVersion); - this.newCredentialsConsumer = Objects.requireNonNull(newCredentialsConsumer); - this.alternativeCredentialsEndpoints = Objects.requireNonNull(alternativeCredentialsEndpoints); - } - - protected HttpHandler createHandler() { - return new Ec2ImdsHttpHandler(ec2ImdsVersion, newCredentialsConsumer, alternativeCredentialsEndpoints); + public Ec2ImdsHttpFixture(Ec2ImdsServiceBuilder ec2ImdsServiceBuilder) { + this.ec2ImdsServiceBuilder = ec2ImdsServiceBuilder; } public String getAddress() { @@ -52,7 +36,7 @@ public void stop(int delay) { protected void before() throws Throwable { server = HttpServer.create(resolveAddress(), 0); - server.createContext("/", Objects.requireNonNull(createHandler())); + server.createContext("/", Objects.requireNonNull(ec2ImdsServiceBuilder.buildHandler())); server.start(); } diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java index 281465b96de05..fd2044357257b 100644 --- a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java @@ -26,6 +26,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.BiConsumer; +import java.util.function.Supplier; import static org.elasticsearch.test.ESTestCase.randomIdentifier; import static org.elasticsearch.test.ESTestCase.randomSecretKey; @@ -43,15 +44,18 @@ public class Ec2ImdsHttpHandler implements HttpHandler { private final BiConsumer newCredentialsConsumer; private final Set validCredentialsEndpoints = ConcurrentCollections.newConcurrentSet(); + private final Supplier availabilityZoneSupplier; public Ec2ImdsHttpHandler( Ec2ImdsVersion ec2ImdsVersion, BiConsumer newCredentialsConsumer, - Collection alternativeCredentialsEndpoints + Collection alternativeCredentialsEndpoints, + Supplier availabilityZoneSupplier ) { this.ec2ImdsVersion = Objects.requireNonNull(ec2ImdsVersion); this.newCredentialsConsumer = Objects.requireNonNull(newCredentialsConsumer); this.validCredentialsEndpoints.addAll(alternativeCredentialsEndpoints); + this.availabilityZoneSupplier = availabilityZoneSupplier; } @Override @@ -98,6 +102,13 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); exchange.getResponseBody().write(response); return; + } else if (path.equals("/latest/meta-data/placement/availability-zone")) { + final var availabilityZone = availabilityZoneSupplier.get(); + final byte[] response = availabilityZone.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + return; } else if (validCredentialsEndpoints.contains(path)) { final String accessKey = randomIdentifier(); final String sessionToken = randomIdentifier(); diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java new file mode 100644 index 0000000000000..bca43da8683b6 --- /dev/null +++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsServiceBuilder.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package fixture.aws.imds; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Collection; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +public class Ec2ImdsServiceBuilder { + + private final Ec2ImdsVersion ec2ImdsVersion; + private BiConsumer newCredentialsConsumer = Ec2ImdsServiceBuilder::rejectNewCredentials; + private Collection alternativeCredentialsEndpoints = Set.of(); + private Supplier availabilityZoneSupplier = Ec2ImdsServiceBuilder::rejectAvailabilityZone; + + public Ec2ImdsServiceBuilder(Ec2ImdsVersion ec2ImdsVersion) { + this.ec2ImdsVersion = ec2ImdsVersion; + } + + public Ec2ImdsServiceBuilder newCredentialsConsumer(BiConsumer newCredentialsConsumer) { + this.newCredentialsConsumer = newCredentialsConsumer; + return this; + } + + private static void rejectNewCredentials(String ignored1, String ignored2) { + ESTestCase.fail("credentials creation not supported"); + } + + public Ec2ImdsServiceBuilder alternativeCredentialsEndpoints(Collection alternativeCredentialsEndpoints) { + this.alternativeCredentialsEndpoints = alternativeCredentialsEndpoints; + return this; + } + + private static String rejectAvailabilityZone() { + return ESTestCase.fail(null, "availability zones not supported"); + } + + public Ec2ImdsServiceBuilder availabilityZoneSupplier(Supplier availabilityZoneSupplier) { + this.availabilityZoneSupplier = availabilityZoneSupplier; + return this; + } + + public Ec2ImdsHttpHandler buildHandler() { + return new Ec2ImdsHttpHandler(ec2ImdsVersion, newCredentialsConsumer, alternativeCredentialsEndpoints, availabilityZoneSupplier); + } + +} diff --git a/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java index bb613395a0fba..6d3eb3d14e9b2 100644 --- a/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java +++ b/test/fixtures/ec2-imds-fixture/src/test/java/fixture/aws/imds/Ec2ImdsHttpHandlerTests.java @@ -30,6 +30,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,7 +44,7 @@ public class Ec2ImdsHttpHandlerTests extends ESTestCase { public void testImdsV1() throws IOException { final Map generatedCredentials = new HashMap<>(); - final var handler = new Ec2ImdsHttpHandler(Ec2ImdsVersion.V1, generatedCredentials::put, Set.of()); + final var handler = new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(generatedCredentials::put).buildHandler(); final var roleResponse = handleRequest(handler, "GET", SECURITY_CREDENTIALS_URI); assertEquals(RestStatus.OK, roleResponse.status()); @@ -66,18 +67,14 @@ public void testImdsV1() throws IOException { public void testImdsV2Disabled() { assertEquals( RestStatus.METHOD_NOT_ALLOWED, - handleRequest( - new Ec2ImdsHttpHandler(Ec2ImdsVersion.V1, (accessKey, sessionToken) -> fail(), Set.of()), - "PUT", - "/latest/api/token" - ).status() + handleRequest(new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).buildHandler(), "PUT", "/latest/api/token").status() ); } public void testImdsV2() throws IOException { final Map generatedCredentials = new HashMap<>(); - final var handler = new Ec2ImdsHttpHandler(Ec2ImdsVersion.V2, generatedCredentials::put, Set.of()); + final var handler = new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(generatedCredentials::put).buildHandler(); final var tokenResponse = handleRequest(handler, "PUT", "/latest/api/token"); assertEquals(RestStatus.OK, tokenResponse.status()); @@ -101,6 +98,21 @@ public void testImdsV2() throws IOException { assertEquals(sessionToken, responseMap.get("Token")); } + public void testAvailabilityZone() { + final Set generatedAvailabilityZones = new HashSet<>(); + final var handler = new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).availabilityZoneSupplier(() -> { + final var newAvailabilityZone = randomIdentifier(); + generatedAvailabilityZones.add(newAvailabilityZone); + return newAvailabilityZone; + }).buildHandler(); + + final var availabilityZoneResponse = handleRequest(handler, "GET", "/latest/meta-data/placement/availability-zone"); + assertEquals(RestStatus.OK, availabilityZoneResponse.status()); + final var availabilityZone = availabilityZoneResponse.body().utf8ToString(); + + assertEquals(generatedAvailabilityZones, Set.of(availabilityZone)); + } + private record TestHttpResponse(RestStatus status, BytesReference body) {} private static TestHttpResponse checkImdsV2GetRequest(Ec2ImdsHttpHandler handler, String uri, String token) { From a0f64d2c9d6cefc0f53d67673db89d35e6d83dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3zala?= <377355+jozala@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:13:58 +0100 Subject: [PATCH 19/24] [ci] Add Alma Linux 9 to matrix in packaging and platform jobs (#118331) SmbTestContainer base image upgraded from Ubuntu 16.04 to 24.04 to avoid hanging Python module compilation when installing samba package. Installing SMB had to be moved from container building to starting because SYS_ADMIN capability is required. --- .../pipelines/periodic-packaging.template.yml | 1 + .buildkite/pipelines/periodic-packaging.yml | 1 + .../pipelines/periodic-platform-support.yml | 1 + .../pull-request/packaging-tests-unix.yml | 1 + x-pack/test/smb-fixture/build.gradle | 2 ++ .../test/fixtures/smb/SmbTestContainer.java | 28 +++++++++++++++---- .../resources/smb/provision/installsmb.sh | 2 +- 7 files changed, 29 insertions(+), 7 deletions(-) mode change 100644 => 100755 x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index aff0add62a2b6..b00ef51ec8f07 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -18,6 +18,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 9bcd61ac1273c..c58201258fbbf 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -19,6 +19,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 8bee3a78f8316..33e9a040b22cf 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -18,6 +18,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 agents: provider: gcp image: family/elasticsearch-{{matrix.image}} diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index 6c7dadfd454ed..3b7973d05a338 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -21,6 +21,7 @@ steps: - rhel-8 - rhel-9 - almalinux-8 + - almalinux-9 PACKAGING_TASK: - docker - docker-cloud-ess diff --git a/x-pack/test/smb-fixture/build.gradle b/x-pack/test/smb-fixture/build.gradle index aeb5626ce9508..a982259edb2dd 100644 --- a/x-pack/test/smb-fixture/build.gradle +++ b/x-pack/test/smb-fixture/build.gradle @@ -2,6 +2,8 @@ apply plugin: 'elasticsearch.java' apply plugin: 'elasticsearch.cache-test-fixtures' dependencies { + implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + api project(':test:fixtures:testcontainer-utils') api "junit:junit:${versions.junit}" api "org.testcontainers:testcontainers:${versions.testcontainer}" diff --git a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java index 10f589e4e1df3..27d8257f4be10 100644 --- a/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java +++ b/x-pack/test/smb-fixture/src/main/java/org/elasticsearch/test/fixtures/smb/SmbTestContainer.java @@ -7,12 +7,18 @@ package org.elasticsearch.test.fixtures.smb; +import com.github.dockerjava.api.model.Capability; + import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.images.builder.ImageFromDockerfile; +import java.time.Duration; + public final class SmbTestContainer extends DockerEnvironmentAwareTestContainer { - private static final String DOCKER_BASE_IMAGE = "ubuntu:16.04"; + private static final String DOCKER_BASE_IMAGE = "ubuntu:24.04"; public static final int AD_LDAP_PORT = 636; public static final int AD_LDAP_GC_PORT = 3269; @@ -20,15 +26,15 @@ public SmbTestContainer() { super( new ImageFromDockerfile("es-smb-fixture").withDockerfileFromBuilder( builder -> builder.from(DOCKER_BASE_IMAGE) - .run("apt-get update -qqy && apt-get install -qqy samba ldap-utils") + .env("TZ", "Etc/UTC") + .run("DEBIAN_FRONTEND=noninteractive apt-get update -qqy && apt-get install -qqy tzdata winbind samba ldap-utils") .copy("fixture/provision/installsmb.sh", "/fixture/provision/installsmb.sh") .copy("fixture/certs/ca.key", "/fixture/certs/ca.key") .copy("fixture/certs/ca.pem", "/fixture/certs/ca.pem") .copy("fixture/certs/cert.pem", "/fixture/certs/cert.pem") .copy("fixture/certs/key.pem", "/fixture/certs/key.pem") .run("chmod +x /fixture/provision/installsmb.sh") - .run("/fixture/provision/installsmb.sh") - .cmd("service samba-ad-dc restart && sleep infinity") + .cmd("/fixture/provision/installsmb.sh && service samba-ad-dc restart && echo Samba started && sleep infinity") .build() ) .withFileFromClasspath("fixture/provision/installsmb.sh", "/smb/provision/installsmb.sh") @@ -37,10 +43,20 @@ public SmbTestContainer() { .withFileFromClasspath("fixture/certs/cert.pem", "/smb/certs/cert.pem") .withFileFromClasspath("fixture/certs/key.pem", "/smb/certs/key.pem") ); - // addExposedPort(389); - // addExposedPort(3268); + addExposedPort(AD_LDAP_PORT); addExposedPort(AD_LDAP_GC_PORT); + + setWaitStrategy( + new WaitAllStrategy().withStartupTimeout(Duration.ofSeconds(120)) + .withStrategy(Wait.forLogMessage(".*Samba started.*", 1)) + .withStrategy(Wait.forListeningPort()) + ); + + getCreateContainerCmdModifiers().add(createContainerCmd -> { + createContainerCmd.getHostConfig().withCapAdd(Capability.SYS_ADMIN); + return createContainerCmd; + }); } public String getAdLdapUrl() { diff --git a/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh b/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh old mode 100644 new mode 100755 index 463238b9f50c2..fe939431bb435 --- a/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh +++ b/x-pack/test/smb-fixture/src/main/resources/smb/provision/installsmb.sh @@ -21,7 +21,7 @@ cat $SSL_DIR/ca.pem >> /etc/ssl/certs/ca-certificates.crt mv /etc/samba/smb.conf /etc/samba/smb.conf.orig -samba-tool domain provision --server-role=dc --use-rfc2307 --dns-backend=SAMBA_INTERNAL --realm=AD.TEST.ELASTICSEARCH.COM --domain=ADES --adminpass=Passw0rd --use-ntvfs +samba-tool domain provision --server-role=dc --use-rfc2307 --dns-backend=SAMBA_INTERNAL --realm=AD.TEST.ELASTICSEARCH.COM --domain=ADES --adminpass=Passw0rd cp /var/lib/samba/private/krb5.conf /etc/krb5.conf From 8c10f0cc384c99718113f0a98542a809dd34ce75 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:15:17 +0100 Subject: [PATCH 20/24] Changes elser service to elasticsearch service in the Semantic search with the inference API page (#118536) --- .../search/search-your-data/semantic-search-inference.asciidoc | 2 +- .../tab-widgets/inference-api/infer-api-requirements.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index 0abc44c809d08..c2fcb88380f53 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -45,7 +45,7 @@ include::{es-ref-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[] ==== Create the index mapping The mapping of the destination index - the index that contains the embeddings that the model will create based on your input text - must be created. -The destination index must have a field with the <> field type for most models and the <> field type for the sparse vector models like in the case of the `elser` service to index the output of the used model. +The destination index must have a field with the <> field type for most models and the <> field type for the sparse vector models like in the case of the `elasticsearch` service to index the output of the used model. include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc index eeecb4718658a..9e935f79aa0ac 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc @@ -8,7 +8,7 @@ the Cohere service. // tag::elser[] ELSER is a model trained by Elastic. If you have an {es} deployment, there is no -further requirement for using the {infer} API with the `elser` service. +further requirement for using the {infer} API with the `elasticsearch` service. // end::elser[] From b2998378a33d41510a52c9efcb22d238cf10b475 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:15:28 +0100 Subject: [PATCH 21/24] [DOCS] Adds default inference endpoints information (#118463) * Adds default inference andpoints information * Update docs/reference/inference/inference-apis.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --------- Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- .../inference/inference-apis.asciidoc | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index c7b779a994a05..8d5ee1b7d6ba5 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -48,21 +48,21 @@ When adaptive allocations are enabled: For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation. -//[discrete] -//[[default-enpoints]] -//=== Default {infer} endpoints +[discrete] +[[default-enpoints]] +=== Default {infer} endpoints -//Your {es} deployment contains some preconfigured {infer} endpoints that makes it easier for you to use them when defining `semantic_text` fields or {infer} processors. -//The following list contains the default {infer} endpoints listed by `inference_id`: +Your {es} deployment contains preconfigured {infer} endpoints which makes them easier to use when defining `semantic_text` fields or using {infer} processors. +The following list contains the default {infer} endpoints listed by `inference_id`: -//* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts) -//* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts) +* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts) +* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts) -//Use the `inference_id` of the endpoint in a <> field definition or when creating an <>. -//The API call will automatically download and deploy the model which might take a couple of minutes. -//Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled. -//For these models, the minimum number of allocations is `0`. -//If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes. +Use the `inference_id` of the endpoint in a <> field definition or when creating an <>. +The API call will automatically download and deploy the model which might take a couple of minutes. +Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled. +For these models, the minimum number of allocations is `0`. +If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes. [discrete] From f3dc0bdd503600723e3dc65e9bf1a321ad94e0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 12 Dec 2024 13:10:56 +0100 Subject: [PATCH 22/24] [Entitlements] Differentiate between ES modules and plugins (external) (#117973) --- libs/entitlement/qa/build.gradle | 4 +- .../entitlement/qa/EntitlementsAllowedIT.java | 4 +- .../bootstrap/EntitlementBootstrap.java | 11 ++--- .../EntitlementInitialization.java | 21 +++++---- .../runtime/policy/ExternalEntitlement.java | 8 ++++ .../runtime/policy/FileEntitlement.java | 2 +- .../runtime/policy/PolicyParser.java | 8 +++- .../policy/PolicyParserFailureTests.java | 21 ++++++--- .../runtime/policy/PolicyParserTests.java | 14 +++++- .../bootstrap/Elasticsearch.java | 14 +++--- .../elasticsearch/plugins/PluginsLoader.java | 44 +++++++++---------- .../plugins/MockPluginsService.java | 2 +- 12 files changed, 96 insertions(+), 57 deletions(-) diff --git a/libs/entitlement/qa/build.gradle b/libs/entitlement/qa/build.gradle index 86bafc34f4d00..7f46b2fe20a8a 100644 --- a/libs/entitlement/qa/build.gradle +++ b/libs/entitlement/qa/build.gradle @@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-test-artifact' dependencies { javaRestTestImplementation project(':libs:entitlement:qa:common') - clusterPlugins project(':libs:entitlement:qa:entitlement-allowed') - clusterPlugins project(':libs:entitlement:qa:entitlement-allowed-nonmodular') + clusterModules project(':libs:entitlement:qa:entitlement-allowed') + clusterModules project(':libs:entitlement:qa:entitlement-allowed-nonmodular') clusterPlugins project(':libs:entitlement:qa:entitlement-denied') clusterPlugins project(':libs:entitlement:qa:entitlement-denied-nonmodular') } diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java index 5135fff44531a..2fd4472f5cc65 100644 --- a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java +++ b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java @@ -28,8 +28,8 @@ public class EntitlementsAllowedIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .plugin("entitlement-allowed") - .plugin("entitlement-allowed-nonmodular") + .module("entitlement-allowed") + .module("entitlement-allowed-nonmodular") .systemProperty("es.entitlements.enabled", "true") .setting("xpack.security.enabled", "false") .build(); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java index 01b8f4d574f90..2abfb11964a93 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java @@ -15,7 +15,6 @@ import com.sun.tools.attach.VirtualMachine; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.Tuple; import org.elasticsearch.entitlement.initialization.EntitlementInitialization; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -29,7 +28,9 @@ public class EntitlementBootstrap { - public record BootstrapArgs(Collection> pluginData, Function, String> pluginResolver) {} + public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {} + + public record BootstrapArgs(Collection pluginData, Function, String> pluginResolver) {} private static BootstrapArgs bootstrapArgs; @@ -40,11 +41,11 @@ public static BootstrapArgs bootstrapArgs() { /** * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}. - * @param pluginData a collection of (plugin path, boolean), that holds the paths of all the installed Elasticsearch modules and - * plugins, and whether they are Java modular or not. + * @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules + * and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins. * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name). */ - public static void bootstrap(Collection> pluginData, Function, String> pluginResolver) { + public static void bootstrap(Collection pluginData, Function, String> pluginResolver) { logger.debug("Loading entitlement agent"); if (EntitlementBootstrap.bootstrapArgs != null) { throw new IllegalStateException("plugin data is already set"); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index fb694308466c6..2956efa8eec31 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -9,7 +9,6 @@ package org.elasticsearch.entitlement.initialization; -import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.provider.ProviderLocator; import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; import org.elasticsearch.entitlement.bridge.EntitlementChecker; @@ -96,25 +95,25 @@ private static PolicyManager createPolicyManager() throws IOException { return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver()); } - private static Map createPluginPolicies(Collection> pluginData) throws IOException { + private static Map createPluginPolicies(Collection pluginData) throws IOException { Map pluginPolicies = new HashMap<>(pluginData.size()); - for (Tuple entry : pluginData) { - Path pluginRoot = entry.v1(); - boolean isModular = entry.v2(); - + for (var entry : pluginData) { + Path pluginRoot = entry.pluginPath(); String pluginName = pluginRoot.getFileName().toString(); - final Policy policy = loadPluginPolicy(pluginRoot, isModular, pluginName); + + final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin()); pluginPolicies.put(pluginName, policy); } return pluginPolicies; } - private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName) throws IOException { + private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin) + throws IOException { Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME); final Set moduleNames = getModuleNames(pluginRoot, isModular); - final Policy policy = parsePolicyIfExists(pluginName, policyFile); + final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin); // TODO: should this check actually be part of the parser? for (Scope scope : policy.scopes) { @@ -125,9 +124,9 @@ private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, Strin return policy; } - private static Policy parsePolicyIfExists(String pluginName, Path policyFile) throws IOException { + private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException { if (Files.exists(policyFile)) { - return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName).parsePolicy(); + return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy(); } return new Policy(pluginName, List.of()); } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java index bb1205696b49e..768babdb840f5 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java @@ -33,4 +33,12 @@ * have to match the parameter names of the constructor. */ String[] parameterNames() default {}; + + /** + * This flag indicates if this Entitlement can be used in external plugins, + * or if it can be used only in Elasticsearch modules ("internal" plugins). + * Using an entitlement that is not {@code pluginsAccessible} in an external + * plugin policy will throw in exception while parsing. + */ + boolean esModulesOnly() default true; } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java index d0837bc096183..4fdbcc93ea6e0 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java @@ -26,7 +26,7 @@ public class FileEntitlement implements Entitlement { private final String path; private final int actions; - @ExternalEntitlement(parameterNames = { "path", "actions" }) + @ExternalEntitlement(parameterNames = { "path", "actions" }, esModulesOnly = false) public FileEntitlement(String path, List actionsList) { this.path = path; int actionsInt = 0; diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index 0d1a7c14ece4b..fb63d5ffbeb48 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -39,6 +39,7 @@ public class PolicyParser { protected final XContentParser policyParser; protected final String policyName; + private final boolean isExternalPlugin; static String getEntitlementTypeName(Class entitlementClass) { var entitlementClassName = entitlementClass.getSimpleName(); @@ -56,9 +57,10 @@ static String getEntitlementTypeName(Class entitlementCla .collect(Collectors.joining("_")); } - public PolicyParser(InputStream inputStream, String policyName) throws IOException { + public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException { this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream)); this.policyName = policyName; + this.isExternalPlugin = isExternalPlugin; } public Policy parsePolicy() { @@ -125,6 +127,10 @@ protected Entitlement parseEntitlement(String scopeName, String entitlementType) throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]"); } + if (entitlementMetadata.esModulesOnly() && isExternalPlugin) { + throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules"); + } + Class[] parameterTypes = entitlementConstructor.getParameterTypes(); String[] parametersNames = entitlementMetadata.parameterNames(); diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java index 7eb2b1fb476b3..dfcc5d8916f2c 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java @@ -19,7 +19,7 @@ public class PolicyParserFailureTests extends ESTestCase { public void testParserSyntaxFailures() { PolicyParserException ppe = expectThrows( PolicyParserException.class, - () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml") + () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false) .parsePolicy() ); assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object ", ppe.getMessage()); @@ -29,7 +29,7 @@ public void testEntitlementDoesNotExist() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - does_not_exist: {} - """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: " + "unknown entitlement type [does_not_exist]", @@ -41,7 +41,7 @@ public void testEntitlementMissingParameter() { PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - file: {} - """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( "[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + "for entitlement type [file]: missing entitlement parameter [path]", @@ -52,7 +52,7 @@ public void testEntitlementMissingParameter() { entitlement-module-name: - file: path: test-path - """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( "[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + "for entitlement type [file]: missing entitlement parameter [actions]", @@ -68,11 +68,22 @@ public void testEntitlementExtraneousParameter() { actions: - read extra: test - """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy()); + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy()); assertEquals( "[7:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] " + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}", ppe.getMessage() ); } + + public void testEntitlementIsNotForExternalPlugins() { + PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - create_class_loader + """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", true).parsePolicy()); + assertEquals( + "[2:5] policy parsing error for [test-failure-policy.yaml]: entitlement type [create_class_loader] is allowed only on modules", + ppe.getMessage() + ); + } } diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index a514cfe418895..633c76cb8c04f 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -37,7 +37,17 @@ public void testGetEntitlementTypeName() { } public void testPolicyBuilder() throws IOException { - Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml") + Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false) + .parsePolicy(); + Policy builtPolicy = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write"))))) + ); + assertEquals(parsedPolicy, builtPolicy); + } + + public void testPolicyBuilderOnExternalPlugin() throws IOException { + Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true) .parsePolicy(); Policy builtPolicy = new Policy( "test-policy.yaml", @@ -50,7 +60,7 @@ public void testParseCreateClassloader() throws IOException { Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: - create_class_loader - """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy(); + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); Policy builtPolicy = new Policy( "test-policy.yaml", List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement()))) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index ae59f6578f03a..9be23c91db072 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -30,7 +30,6 @@ import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.Tuple; import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexVersion; @@ -59,6 +58,7 @@ import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static org.elasticsearch.bootstrap.BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING; import static org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler.CTRL_CLOSE_EVENT; @@ -218,10 +218,14 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) { LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements"); - List> pluginData = pluginsLoader.allBundles() - .stream() - .map(bundle -> Tuple.tuple(bundle.getDir(), bundle.pluginDescriptor().isModular())) - .toList(); + List pluginData = Stream.concat( + pluginsLoader.moduleBundles() + .stream() + .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)), + pluginsLoader.pluginBundles() + .stream() + .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true)) + ).toList(); EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName); } else if (RuntimeVersionFeature.isSecurityManagerAvailable()) { diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java index 8dfc1fc27c6aa..c7dc2c405ffba 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java @@ -122,7 +122,8 @@ public static LayerAndLoader ofUberModuleLoader(UberModuleClassLoader loader) { private final List moduleDescriptors; private final List pluginDescriptors; private final Map loadedPluginLayers; - private final Set allBundles; + private final Set moduleBundles; + private final Set pluginBundles; /** * Constructs a new PluginsLoader @@ -153,37 +154,36 @@ public static PluginsLoader createPluginsLoader(Path modulesDirectory, Path plug Set seenBundles = new LinkedHashSet<>(); // load (elasticsearch) module layers - List moduleDescriptors; + final Set modules; if (modulesDirectory != null) { try { - Set modules = PluginsUtils.getModuleBundles(modulesDirectory); - moduleDescriptors = modules.stream().map(PluginBundle::pluginDescriptor).toList(); + modules = PluginsUtils.getModuleBundles(modulesDirectory); seenBundles.addAll(modules); } catch (IOException ex) { throw new IllegalStateException("Unable to initialize modules", ex); } } else { - moduleDescriptors = Collections.emptyList(); + modules = Collections.emptySet(); } // load plugin layers - List pluginDescriptors; + final Set plugins; if (pluginsDirectory != null) { try { // TODO: remove this leniency, but tests bogusly rely on it if (isAccessibleDirectory(pluginsDirectory, logger)) { PluginsUtils.checkForFailedPluginRemovals(pluginsDirectory); - Set plugins = PluginsUtils.getPluginBundles(pluginsDirectory); - pluginDescriptors = plugins.stream().map(PluginBundle::pluginDescriptor).toList(); + plugins = PluginsUtils.getPluginBundles(pluginsDirectory); + seenBundles.addAll(plugins); } else { - pluginDescriptors = Collections.emptyList(); + plugins = Collections.emptySet(); } } catch (IOException ex) { throw new IllegalStateException("Unable to initialize plugins", ex); } } else { - pluginDescriptors = Collections.emptyList(); + plugins = Collections.emptySet(); } Map loadedPluginLayers = new LinkedHashMap<>(); @@ -197,19 +197,15 @@ public static PluginsLoader createPluginsLoader(Path modulesDirectory, Path plug } } - return new PluginsLoader(moduleDescriptors, pluginDescriptors, loadedPluginLayers, Set.copyOf(seenBundles)); + return new PluginsLoader(modules, plugins, loadedPluginLayers); } - PluginsLoader( - List moduleDescriptors, - List pluginDescriptors, - Map loadedPluginLayers, - Set allBundles - ) { - this.moduleDescriptors = moduleDescriptors; - this.pluginDescriptors = pluginDescriptors; + PluginsLoader(Set modules, Set plugins, Map loadedPluginLayers) { + this.moduleBundles = modules; + this.pluginBundles = plugins; + this.moduleDescriptors = modules.stream().map(PluginBundle::pluginDescriptor).toList(); + this.pluginDescriptors = plugins.stream().map(PluginBundle::pluginDescriptor).toList(); this.loadedPluginLayers = loadedPluginLayers; - this.allBundles = allBundles; } public List moduleDescriptors() { @@ -224,8 +220,12 @@ public Stream pluginLayers() { return loadedPluginLayers.values().stream().map(Function.identity()); } - public Set allBundles() { - return allBundles; + public Set moduleBundles() { + return moduleBundles; + } + + public Set pluginBundles() { + return pluginBundles; } private static void loadPluginLayer( diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java index 91875600ec000..0a4c99eb8b52a 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java @@ -45,7 +45,7 @@ public MockPluginsService(Settings settings, Environment environment, Collection super( settings, environment.configFile(), - new PluginsLoader(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), Collections.emptySet()) + new PluginsLoader(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap()) ); List pluginsLoaded = new ArrayList<>(); From d51431535f0d4ae2acf5059ace9c4d5ba8e598d2 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 12 Dec 2024 12:34:00 +0000 Subject: [PATCH 23/24] Remove DiscoveryNodes.getSmallestNonClientNodeVersion method (#116508) --- .../cluster/node/DiscoveryNodes.java | 18 ------------------ .../SystemIndexMappingUpdateService.java | 2 +- .../cluster/node/DiscoveryNodesTests.java | 1 - .../license/TransportPostStartTrialAction.java | 2 +- 4 files changed, 2 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 12c698a6ed958..5e6dec7b68062 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -65,7 +65,6 @@ public class DiscoveryNodes implements Iterable, SimpleDiffable ingestNodes, @Nullable String masterNodeId, @Nullable String localNodeId, - Version minNonClientNodeVersion, Version maxNodeVersion, Version minNodeVersion, IndexVersion maxDataNodeCompatibleIndexVersion, @@ -98,7 +96,6 @@ private DiscoveryNodes( assert (masterNodeId == null) == (masterNode == null); this.localNodeId = localNodeId; this.localNode = localNodeId == null ? null : nodes.get(localNodeId); - this.minNonClientNodeVersion = minNonClientNodeVersion; this.minNodeVersion = minNodeVersion; this.maxNodeVersion = maxNodeVersion; this.maxDataNodeCompatibleIndexVersion = maxDataNodeCompatibleIndexVersion; @@ -117,7 +114,6 @@ public DiscoveryNodes withMasterNodeId(@Nullable String masterNodeId) { ingestNodes, masterNodeId, localNodeId, - minNonClientNodeVersion, maxNodeVersion, minNodeVersion, maxDataNodeCompatibleIndexVersion, @@ -346,17 +342,6 @@ public boolean isMixedVersionCluster() { return minNodeVersion.equals(maxNodeVersion) == false; } - /** - * Returns the version of the node with the oldest version in the cluster that is not a client node - * - * If there are no non-client nodes, Version.CURRENT will be returned. - * - * @return the oldest version in the cluster - */ - public Version getSmallestNonClientNodeVersion() { - return minNonClientNodeVersion; - } - /** * Returns the highest index version supported by all data nodes in the cluster */ @@ -853,14 +838,12 @@ public DiscoveryNodes build() { */ Version minNodeVersion = null; Version maxNodeVersion = null; - Version minNonClientNodeVersion = null; IndexVersion maxDataNodeCompatibleIndexVersion = null; IndexVersion minSupportedIndexVersion = null; for (Map.Entry nodeEntry : nodes.entrySet()) { DiscoveryNode discoNode = nodeEntry.getValue(); Version version = discoNode.getVersion(); if (discoNode.canContainData() || discoNode.isMasterNode()) { - minNonClientNodeVersion = min(minNonClientNodeVersion, version); maxDataNodeCompatibleIndexVersion = min(maxDataNodeCompatibleIndexVersion, discoNode.getMaxIndexVersion()); } minNodeVersion = min(minNodeVersion, version); @@ -894,7 +877,6 @@ public DiscoveryNodes build() { filteredNodes(nodes, DiscoveryNode::isIngestNode), masterNodeId, localNodeId, - Objects.requireNonNullElse(minNonClientNodeVersion, Version.CURRENT), Objects.requireNonNullElse(maxNodeVersion, Version.CURRENT), Objects.requireNonNullElse(minNodeVersion, Version.CURRENT.minimumCompatibilityVersion()), Objects.requireNonNullElse(maxDataNodeCompatibleIndexVersion, IndexVersion.current()), diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java index f65ed270e29c4..a00e8cdc138f6 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java @@ -93,7 +93,7 @@ public void clusterChanged(ClusterChangedEvent event) { } // if we're in a mixed-version cluster, exit - if (state.nodes().getMaxNodeVersion().after(state.nodes().getSmallestNonClientNodeVersion())) { + if (state.nodes().isMixedVersionCluster()) { logger.debug("Skipping system indices up-to-date check as cluster has mixed versions"); return; } diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java index 3264cf168b638..5101064f293e4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java @@ -435,7 +435,6 @@ public void testMinMaxNodeVersions() { assertEquals(Version.fromString("5.0.17"), build.getMaxNodeVersion()); assertEquals(Version.fromString("1.6.0"), build.getMinNodeVersion()); - assertEquals(Version.fromString("2.1.0"), build.getSmallestNonClientNodeVersion()); // doesn't include 1.6.0 observer assertEquals(IndexVersion.fromId(2010099), build.getMaxDataNodeCompatibleIndexVersion()); // doesn't include 2000199 observer assertEquals(IndexVersion.fromId(2000099), build.getMinSupportedIndexVersion()); // also includes observers } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java index 8a7ead2caf3be..212e593c9db98 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java @@ -55,7 +55,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { - if (state.nodes().getMaxNodeVersion().after(state.nodes().getSmallestNonClientNodeVersion())) { + if (state.nodes().isMixedVersionCluster()) { throw new IllegalStateException( "Please ensure all nodes are on the same version before starting your trial, the highest node version in this cluster is [" + state.nodes().getMaxNodeVersion() From e7a4436e27d1295c842df6fd096251defbdb0c26 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 12 Dec 2024 14:29:33 +0100 Subject: [PATCH 24/24] ESQL: Disable remote enrich verification (#118534) This disables verifying the plans generated for remote ENRICHing. It also re-enables corresponding failing test. Related: #118531 Fixes #118307. --- muted-tests.yml | 3 --- .../xpack/esql/optimizer/PhysicalVerifier.java | 8 ++++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index c91c7b50a0808..b750c0777ce34 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -276,9 +276,6 @@ tests: - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests method: testBottomFieldSort issue: https://github.com/elastic/elasticsearch/issues/118214 -- class: org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT - method: testTopNThenEnrichRemote - issue: https://github.com/elastic/elasticsearch/issues/118307 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1UnavailableRemotesIT method: testEsqlRcs1UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/118350 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java index 20528f8dc2826..9132cf87541bb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java @@ -12,7 +12,9 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.optimizer.rules.PlanConsistencyChecker; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; +import org.elasticsearch.xpack.esql.plan.physical.EnrichExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -35,6 +37,12 @@ public Collection verify(PhysicalPlan plan) { Set failures = new LinkedHashSet<>(); Failures depFailures = new Failures(); + // AwaitsFix https://github.com/elastic/elasticsearch/issues/118531 + var enriches = plan.collectFirstChildren(EnrichExec.class::isInstance); + if (enriches.isEmpty() == false && ((EnrichExec) enriches.get(0)).mode() == Enrich.Mode.REMOTE) { + return failures; + } + plan.forEachDown(p -> { if (p instanceof AggregateExec agg) { var exclude = Expressions.references(agg.ordinalAttributes());