Version upgrade to es 8.11.4 (#478)

* [WIP]version upgrade to es 8.11.1

* change to luceneVersion = 9.8.0

* Based on PR 478 (#478), continued upgrade changes, still work in progress.

* Test failures fix trial and error.

* Fix typo

* Fix for failing YAML rest tests, checkstyle changes.

---------

Co-authored-by: Daniel Wrigley <54574577+wrigleyDan@users.noreply.github.com>
Co-authored-by: wrigleyDan <dwrigley@o19s.com>

gradle.properties

@@ -1,5 +1,5 @@
 ltrVersion = 1.5.8
-elasticsearchVersion = 8.9.2
-luceneVersion = 9.7.0
+elasticsearchVersion = 8.11.4
+luceneVersion = 9.8.0
 ow2Version = 8.0.1
 antlrVersion = 4.5.1-1

src/javaRestTest/java/com/o19s/es/ltr/action/ListStoresActionIT.java

@@ -32,6 +32,10 @@
 import static com.o19s.es.ltr.feature.store.index.IndexFeatureStore.indexName;
 
 public class ListStoresActionIT extends BaseIntegrationTest {
+ /*
+ Create two stores (test2, test2) in addition to the default store.
+ List the stores.
+ */
  public void testListStore() throws Exception {
  createStore(indexName("test2"));
  createStore(indexName("test3"));
@@ -54,6 +58,46 @@ public void testListStore() throws Exception {
  }
  }
 
+ /*
+ Create two stores (test2, test2) in addition to the default store.
+ List the stores.
+ Delete a store ("test2") and make sure the store is not listed any more.
+ */
+ public void testListRemovedStore() throws Exception {
+ createStore(indexName("test2"));
+ createStore(indexName("test3"));
+ Map<String, IndexStoreInfo> infos = new HashMap<>();
+ String[] stores = new String[]{IndexFeatureStore.DEFAULT_STORE, indexName("test2"), indexName("test3")};
+ for (String store : stores) {
+ infos.put(IndexFeatureStore.storeName(store),
+ new IndexStoreInfo(store, IndexFeatureStore.VERSION, addElements(store)));
+ }
+ ListStoresActionResponse resp = new ListStoresAction.ListStoresActionBuilder(client()).execute().get();
+ assertEquals(infos.size(), resp.getStores().size());
+ assertEquals(infos.keySet(), resp.getStores().keySet());
+ for (String k : infos.keySet()) {
+ IndexStoreInfo expected = infos.get(k);
+ IndexStoreInfo actual = resp.getStores().get(k);
+ assertEquals(expected.getIndexName(), actual.getIndexName());
+ assertEquals(expected.getStoreName(), actual.getStoreName());
+ assertEquals(expected.getVersion(), actual.getVersion());
+ assertEquals(expected.getCounts(), actual.getCounts());
+ }
+ deleteStore(indexName("test2"));
+ infos.remove("test2");
+ resp = new ListStoresAction.ListStoresActionBuilder(client()).execute().get();
+ assertEquals(infos.size(), resp.getStores().size());
+ assertEquals(infos.keySet(), resp.getStores().keySet());
+ for (String k : infos.keySet()) {
+ IndexStoreInfo expected = infos.get(k);
+ IndexStoreInfo actual = resp.getStores().get(k);
+ assertEquals(expected.getIndexName(), actual.getIndexName());
+ assertEquals(expected.getStoreName(), actual.getStoreName());
+ assertEquals(expected.getVersion(), actual.getVersion());
+ assertEquals(expected.getCounts(), actual.getCounts());
+ }
+ }
+
  private Map<String, Integer> addElements(String store) throws Exception {
  Map<String, Integer> counts = new HashMap<>();
  int nFeats = randomInt(20) + 1;

src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java

@@ -27,7 +27,7 @@
 import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.tests.util.TestUtil;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
@@ -715,7 +715,7 @@ public Map<String,Doc> buildIndex() {
  }
 
  public void indexDoc(Doc d) {
- IndexResponse resp = client().prepareIndex("test_index")
+ DocWriteResponse resp = client().prepareIndex("test_index")
  .setSource("field1", d.field1, "field2", d.field2, "scorefield1", d.scorefield1, "nesteddocs1", d.getNesteddocs1())
  .get();
  d.id = resp.getId();

src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java

@@ -16,6 +16,7 @@
 package com.o19s.es.explore;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
@@ -157,6 +158,6 @@ public ExplorerQueryBuilder statsType(String type) {
 
  @Override
  public TransportVersion getMinimalSupportedVersion() {
- return TransportVersion.V_7_0_0;
+ return TransportVersions.V_7_0_0;
  }
 }

src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java

@@ -77,6 +77,8 @@
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
@@ -105,7 +107,6 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
@@ -252,8 +253,9 @@ public Collection<Object> createComponents(Client client,
  NamedWriteableRegistry namedWriteableRegistry,
  IndexNameExpressionResolver indexNameExpressionResolver,
  Supplier<RepositoriesService> repositoriesServiceSupplier,
- Tracer tracer,
- AllocationService allocationService) {
+ TelemetryProvider telemetryProvider,
+ AllocationService allocationService,
+ IndicesService indicesService) {
  clusterService.addListener(event -> {
  for (Index i : event.indicesDeleted()) {
  if (IndexFeatureStore.isIndexStore(i.getName())) {

src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java

@@ -20,18 +20,19 @@
 import com.o19s.es.ltr.feature.store.StoredFeature;
 import com.o19s.es.ltr.feature.FeatureValidation;
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
-import org.elasticsearch.common.xcontent.StatusToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.List;
@@ -73,7 +74,7 @@ public AddFeaturesToSetRequest() {
  public AddFeaturesToSetRequest(StreamInput in) throws IOException {
  super(in);
  store = in.readString();
- features = in.readList(StoredFeature::new);
+ features = in.readCollectionAsList(StoredFeature::new);
  if (in.readBoolean()) {
  featureNameQuery = in.readOptionalString();
  }
@@ -107,7 +108,7 @@ public void writeTo(StreamOutput out) throws IOException {
  out.writeString(store);
  out.writeOptionalString(featureNameQuery);
  if (features != null) {
- out.writeList(features);
+ out.writeCollection(features);
  }
  out.writeBoolean(merge);
  out.writeString(featureSet);
@@ -172,15 +173,15 @@ public void setValidation(FeatureValidation validation) {
  }
  }
 
- public static class AddFeaturesToSetResponse extends ActionResponse implements StatusToXContentObject {
- private IndexResponse response;
+ public static class AddFeaturesToSetResponse extends ActionResponse implements ToXContentObject {
+ private DocWriteResponse response;
 
  public AddFeaturesToSetResponse(StreamInput in) throws IOException {
  super(in);
  response = new IndexResponse(in);
  }
 
- public AddFeaturesToSetResponse(IndexResponse response) {
+ public AddFeaturesToSetResponse(DocWriteResponse response) {
  this.response = response;
  }
 
@@ -189,20 +190,19 @@ public void writeTo(StreamOutput out) throws IOException {
  response.writeTo(out);
  }
 
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ return response.toXContent(builder, params);
+ }
+
  /**
  * Returns the REST status to make sure it is returned correctly
  */
- @Override
  public RestStatus status() {
  return response.status();
  }
 
- @Override
- public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
- return response.toXContent(builder, params);
- }
-
- public IndexResponse getResponse() {
+ public DocWriteResponse getResponse() {
  return response;
  }
  }

src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java

@@ -78,12 +78,12 @@ public CachesStatsNodesResponse(ClusterName clusterName, List<CachesStatsNodeRes
 
  @Override
  protected List<CachesStatsNodeResponse> readNodesFrom(StreamInput in) throws IOException {
- return in.readList(CachesStatsNodeResponse::new);
+ return in.readCollectionAsList(CachesStatsNodeResponse::new);
  }
 
  @Override
  protected void writeNodesTo(StreamOutput out, List<CachesStatsNodeResponse> nodes) throws IOException {
- out.writeList(nodes);
+ out.writeCollection(nodes);
  }
 
  @Override

src/main/java/com/o19s/es/ltr/action/ClearCachesAction.java

@@ -153,12 +153,12 @@ public ClearCachesNodesResponse(ClusterName clusterName, List<ClearCachesNodeRes
 
  @Override
  protected List<ClearCachesNodeResponse> readNodesFrom(StreamInput in) throws IOException {
- return in.readList(ClearCachesNodeResponse::new);
+ return in.readCollectionAsList(ClearCachesNodeResponse::new);
  }
 
  @Override
  protected void writeNodesTo(StreamOutput out, List<ClearCachesNodeResponse> nodes) throws IOException {
- out.writeList(nodes);
+ out.writeCollection(nodes);
  }
  }
 

src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java

@@ -20,17 +20,18 @@
 import com.o19s.es.ltr.feature.FeatureValidation;
 import com.o19s.es.ltr.feature.store.StoredLtrModel;
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 
@@ -46,7 +47,7 @@ protected CreateModelFromSetAction() {
 
 
  public static class CreateModelFromSetRequestBuilder extends ActionRequestBuilder<CreateModelFromSetRequest,
- CreateModelFromSetResponse> {
+  CreateModelFromSetResponse> {
 
  public CreateModelFromSetRequestBuilder(ElasticsearchClient client) {
  super(client, INSTANCE, new CreateModelFromSetRequest());
@@ -115,7 +116,7 @@ public ActionRequestValidationException validate() {
  arve = addValidationError("modelName must be set", arve);
  }
  if (definition == null) {
- arve = addValidationError("defition must be set", arve);
+ arve = addValidationError("definition must be set", arve);
  }
  return arve;
  }
@@ -169,9 +170,9 @@ public void setValidation(FeatureValidation validation) {
  }
  }
 
- public static class CreateModelFromSetResponse extends ActionResponse implements StatusToXContentObject {
+ public static class CreateModelFromSetResponse extends ActionResponse implements ToXContentObject {
  private static final int VERSION = 1;
- private IndexResponse response;
+ private DocWriteResponse response;
 
  public CreateModelFromSetResponse(StreamInput in) throws IOException {
  super(in);
@@ -180,7 +181,7 @@ public CreateModelFromSetResponse(StreamInput in) throws IOException {
  response = new IndexResponse(in);
  }
 
- public CreateModelFromSetResponse(IndexResponse response) {
+ public CreateModelFromSetResponse(DocWriteResponse response) {
  this.response = response;
  }
 
@@ -190,7 +191,7 @@ public void writeTo(StreamOutput out) throws IOException {
  response.writeTo(out);
  }
 
- public IndexResponse getResponse() {
+ public DocWriteResponse getResponse() {
  return response;
  }
 
@@ -199,9 +200,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
  return response.toXContent(builder, params);
  }
 
- @Override
+ /**
+ * Returns the REST status to make sure it is returned correctly
+ */
  public RestStatus status() {
  return response.status();
  }
+
  }
 }

src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java

@@ -21,18 +21,19 @@
 import com.o19s.es.ltr.feature.store.StorableElement;
 import com.o19s.es.ltr.feature.store.index.IndexFeatureStore;
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
-import org.elasticsearch.common.xcontent.StatusToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -145,10 +146,6 @@ public void setRouting(String routing) {
  this.routing = routing;
  }
 
- public Long getUpdatedVersion() {
- return updatedVersion;
- }
-
  public FeatureValidation getValidation() {
  return validation;
  }
@@ -173,19 +170,19 @@ public enum Action {
  }
  }
 
- public static class FeatureStoreResponse extends ActionResponse implements StatusToXContentObject {
- private IndexResponse response;
+ public static class FeatureStoreResponse extends ActionResponse implements ToXContentObject {
+ private DocWriteResponse response;
 
  public FeatureStoreResponse(StreamInput in) throws IOException {
  super(in);
  response = new IndexResponse(in);
  }
 
- public FeatureStoreResponse(IndexResponse response) {
+ public FeatureStoreResponse(DocWriteResponse response) {
  this.response = response;
  }
 
- public IndexResponse getResponse() {
+ public DocWriteResponse getResponse() {
  return response;
  }
 
@@ -197,7 +194,6 @@ public void writeTo(StreamOutput out) throws IOException {
  /**
  * Returns the REST status to make sure it is returned correctly
  */
- @Override
  public RestStatus status() {
  return response.status();
  }