Enable HLRC compatibility mode by default

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -347,7 +347,12 @@ protected RestHighLevelClient(
                 .flatMap(Function.identity())
                 .collect(toList())
         );
-        if (useAPICompatibility == null && "true".equals(System.getenv(API_VERSIONING_ENV_VARIABLE))) {
+
+        // Compatibility mode is on by default, then env variable has precedence over builder setting
+        String apiVersioningEnv = System.getenv(API_VERSIONING_ENV_VARIABLE);
+        if (useAPICompatibility == null && apiVersioningEnv == null) {
+            this.useAPICompatibility = true;
+        } else if (useAPICompatibility == null && "true".equals(System.getenv(API_VERSIONING_ENV_VARIABLE))) {
             this.useAPICompatibility = true;
         } else {
             this.useAPICompatibility = Boolean.TRUE.equals(useAPICompatibility);

client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java

@@ -147,7 +147,8 @@ private Void mockPerformRequestAsync(Request request, ResponseListener responseL
      * Mocks the synchronous request execution like if it was executed by Elasticsearch.
      */
     private Response mockPerformRequest(Request request) throws IOException {
-        assertThat(request.getOptions().getHeaders(), hasSize(1));
+        // Headers contain 'node_name' set by optionsForNodeName and 'Accept' from HLRC compatibility mode
+        assertThat(request.getOptions().getHeaders(), hasSize(2));
         Header httpHeader = request.getOptions().getHeaders().get(0);
         final Response mockResponse = mock(Response.class);
         when(mockResponse.getHost()).thenReturn(new HttpHost("localhost", 9200));

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -33,6 +33,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.action.search.SearchResponse;
@@ -95,6 +97,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
 import org.elasticsearch.index.rankeval.EvaluationMetric;
 import org.elasticsearch.index.rankeval.ExpectedReciprocalRank;
@@ -1501,6 +1504,30 @@ public void testModifyForCompatibility() {
 
     }
 
+    public void testCompatibilityModeDefault() throws Exception {
+        mockResponse(new GetResponse(new GetResult("foo", "bar", "1", 1, 1, 1, true, null, null, null)));
+        restHighLevelClient.get(new GetRequest("foo", "bar"), RequestOptions.DEFAULT);
+
+        verify(restClient).performRequest(argThat(req -> {
+            List<Header> headers = req.getOptions().getHeaders();
+            Header accept = headers.stream().filter(h -> h.getName().equals("Accept")).findFirst().get();
+            return accept.getValue().equals("application/vnd.elasticsearch+json; compatible-with=7");
+        }));
+    }
+
+    public void testDisableCompatibilityMode() throws Exception {
+        RestHighLevelClient client = new RestHighLevelClientBuilder(restClient).setApiCompatibilityMode(false).build();
+
+        mockResponse(new GetResponse(new GetResult("foo", "bar", "1", 1, 1, 1, true, null, null, null)));
+        client.get(new GetRequest("foo", "bar"), RequestOptions.DEFAULT);
+
+        verify(restClient).performRequest(argThat(req -> {
+            List<Header> headers = req.getOptions().getHeaders();
+            Optional<Header> accept = headers.stream().filter(h -> h.getName().equals("Accept")).findFirst();
+            return accept.isPresent() == false;
+        }));
+    }
+
     private static void assertSyncMethod(Method method, String apiName, List<String> booleanReturnMethods) {
         // A few methods return a boolean rather than a response object
         if (apiName.equals("ping") || apiName.contains("exist") || booleanReturnMethods.contains(apiName)) {

docs/changelog/86517.yaml

@@ -0,0 +1,18 @@
+pr: 86517
+summary: Enable HLRC compatibility mode by default
+area: Java High Level REST Client
+type: "breaking"
+issues: []
+breaking:
+  title: Enable HLRC compatibility mode by default
+  area: REST API
+  details: |-
+    Compatibility mode allows HLRC to communicate with {es} 8.x by sending version information in the `Content-Type` and `Accept`
+    request headers, which causes {es} 8 to behave like {es} 7. As this feature isn't very well known, a number of users had
+    the wrong impression that HLRC 7.17 was not working with {es} 8.
+
+    Compatibility mode is now enabled by default, allowing it to work out of the box with {es} 8.
+  impact: |-
+    Compatibilty mode was introduced in {es} 7.11. Using HLRC 7.17.4 with {es} 7.10 and before therefore requires to disable
+    compatibility mode when creating an HLRC instance.
+  notable: true

docs/java-rest/high-level/getting-started.asciidoc

@@ -18,11 +18,11 @@ doesn't need to be in the same minor version as the Elasticsearch nodes it
 communicates with, as it is forward compatible meaning that it supports
 communicating with later versions of Elasticsearch than the one it was developed for.
 
-The 6.0 client is able to communicate with any 6.x Elasticsearch node, while the 6.1
-client is for sure able to communicate with 6.1, 6.2 and any later 6.x version, but
+The 7.0 client is able to communicate with any 7.x Elasticsearch node, while the 7.1
+client is for sure able to communicate with 7.1, 7.2 and any later 7.x version, but
 there may be incompatibility issues when communicating with a previous Elasticsearch
-node version, for instance between 6.1 and 6.0, in case the 6.1 client supports new
-request body fields for some APIs that are not known by the 6.0 node(s).
+node version, for instance between 7.1 and 7.0, in case the 7.1 client supports new
+request body fields for some APIs that are not known by the 7.0 node(s).
 
 It is recommended to upgrade the High Level Client when upgrading the Elasticsearch
 cluster to a new major version, as REST API breaking changes may cause unexpected
@@ -31,6 +31,23 @@ only be supported by the newer version of the client. The client should always b
 updated last, once all of the nodes in the cluster have been upgraded to the new
 major version.
 
+*Compatibility with Elasticsearch 8.x*
+
+The High Level Client version 7.16 and higher can communicate with Elasticsearch version 8.x after enabling API compatibility mode. When this mode enabled, the client will send HTTP headers that instruct Elasticsearch 8.x to honor 7.x request/responses.
+
+Compatibility mode is enabled as follows:
+
+["source","java",subs="attributes"]
+--------------------------------------------------
+RestHighLevelClient esClient = new RestHighLevelClientBuilder(restClient)
+    .setApiCompatibilityMode(true)
+    .build()
+--------------------------------------------------
+
+When compatibility mode is enabled, the client can also communicate with Elasticsearch version 7.11 and higher.
+
+NOTE: Starting with version 7.17.4, compatibility mode is enabled by default.
+
 [[java-rest-high-javadoc]]
 === Javadoc
 
@@ -141,7 +158,7 @@ transitive dependencies:
 [[java-rest-high-getting-started-initialization]]
 === Initialization
 
-A `RestHighLevelClient` instance needs a 
+A `RestHighLevelClient` instance needs a
 {java-api-client}/java-rest-low-usage-initialization.html[REST low-level client builder]
 to be built as follows:
 
@@ -172,27 +189,27 @@ All APIs in the `RestHighLevelClient` accept a `RequestOptions` which you can
 use to customize the request in ways that won't change how Elasticsearch
 executes the request. For example, this is the place where you'd specify a
 `NodeSelector` to control which node receives the request. See the
-{java-api-client}/java-rest-low-usage-requests.html#java-rest-low-usage-request-options[low level client documentation] 
+{java-api-client}/java-rest-low-usage-requests.html#java-rest-low-usage-request-options[low level client documentation]
 for more examples of customizing the options.
 
 [[java-rest-high-getting-started-asynchronous-usage]]
 === Asynchronous usage
 
-All of the methods across the different clients exist in a traditional synchronous and 
-asynchronous variant. The difference is that the asynchronous ones use asynchronous requests 
+All of the methods across the different clients exist in a traditional synchronous and
+asynchronous variant. The difference is that the asynchronous ones use asynchronous requests
 in the REST Low Level Client. This is useful if you are doing multiple requests or are using e.g.
 rx java, Kotlin co-routines, or similar frameworks.
 
-The asynchronous methods are recognizable by the fact that they have the word "Async" in their name 
-and return a `Cancellable` instance. The asynchronous methods accept the same request object 
-as the synchronous variant and accept a generic `ActionListener<T>` where `T` is the return 
-type of the synchronous method. 
+The asynchronous methods are recognizable by the fact that they have the word "Async" in their name
+and return a `Cancellable` instance. The asynchronous methods accept the same request object
+as the synchronous variant and accept a generic `ActionListener<T>` where `T` is the return
+type of the synchronous method.
 
-All asynchronous methods return a `Cancellable` object with a `cancel` method that you may call 
+All asynchronous methods return a `Cancellable` object with a `cancel` method that you may call
 in case you want to abort the request. Cancelling
-no longer needed requests is a good way to avoid putting unnecessary 
+no longer needed requests is a good way to avoid putting unnecessary
 load on Elasticsearch.
 
-Using the `Cancellable` instance is optional and you can safely ignore this if you have 
-no need for this. A use case for this would be using this with e.g. Kotlin's `suspendCancellableCoRoutine`. 
+Using the `Cancellable` instance is optional and you can safely ignore this if you have
+no need for this. A use case for this would be using this with e.g. Kotlin's `suspendCancellableCoRoutine`.
 

docs/java-rest/high-level/index.asciidoc

@@ -8,6 +8,8 @@
 
 deprecated[7.15.0, The High Level REST Client is deprecated in favour of the {java-api-client}/index.html[Java API Client].]
 
+NOTE: The High Level Rest Client version 7.17 can work with {es} `8.x` with <<java-rest-high-compatibility,compatibility mode enabled>>.
+
 The Java High Level REST Client works on top of the Java Low Level REST client.
 Its main goal is to expose API specific methods, that accept request objects as
 an argument and return response objects, so that request marshalling and

qa/ccs-old-version-remote-cluster/src/test/java/org/elasticsearch/upgrades/CCSFieldsOptionEmulationIT.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.RestHighLevelClientBuilder;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.document.DocumentField;
@@ -89,11 +90,13 @@ static RestHighLevelClient newLocalClient(Logger logger) {
         final List<HttpHost> hosts = parseHosts("tests.rest.cluster");
         final int index = random().nextInt(hosts.size());
         logger.info("Using client node {}", index);
-        return new RestHighLevelClient(RestClient.builder(hosts.get(index)));
+        return new RestHighLevelClientBuilder(RestClient.builder(hosts.get(index)).build()).setApiCompatibilityMode(false).build();
     }
 
     static RestHighLevelClient newRemoteClient() {
-        return new RestHighLevelClient(RestClient.builder(randomFrom(parseHosts("tests.rest.remote_cluster"))));
+        return new RestHighLevelClientBuilder(RestClient.builder(randomFrom(parseHosts("tests.rest.remote_cluster"))).build())
+            .setApiCompatibilityMode(false)
+            .build();
     }
 
     public void testFieldsOptionEmulation() throws Exception {

qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.RestHighLevelClientBuilder;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
@@ -72,11 +73,13 @@ protected static RestHighLevelClient newLocalClient(Logger logger) {
         final List<HttpHost> hosts = parseHosts("tests.rest.cluster");
         final int index = random().nextInt(hosts.size());
         logger.info("Using client node {}", index);
-        return new RestHighLevelClient(RestClient.builder(hosts.get(index)));
+        return new RestHighLevelClientBuilder(RestClient.builder(hosts.get(index)).build()).setApiCompatibilityMode(false).build();
     }
 
     protected static RestHighLevelClient newRemoteClient() {
-        return new RestHighLevelClient(RestClient.builder(randomFrom(parseHosts("tests.rest.remote_cluster"))));
+        return new RestHighLevelClientBuilder(RestClient.builder(randomFrom(parseHosts("tests.rest.remote_cluster"))).build())
+            .setApiCompatibilityMode(false)
+            .build();
     }
 
     static int indexDocs(RestHighLevelClient client, String index, int numDocs) throws IOException {

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.RestHighLevelClientBuilder;
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -69,10 +70,8 @@ public class MlJobSnapshotUpgradeIT extends AbstractUpgradeTestCase {
     // min version in upgraded 7.series is 7.11.0
     private static final Version CPP_COMPATIBILTIY_VERSION = Version.V_7_11_0;
 
-    private static class HLRC extends RestHighLevelClient {
-        HLRC(RestClient restClient) {
-            super(restClient, RestClient::close, new ArrayList<>());
-        }
+    private static RestHighLevelClient HLRC(RestClient restClient) {
+        return new RestHighLevelClientBuilder(restClient).setApiCompatibilityMode(false).build();
     }
 
     private MachineLearningClient hlrc;
@@ -98,7 +97,7 @@ protected static void waitForPendingUpgraderTasks() throws Exception {
      * index mappings when it is assigned to an upgraded node even if no other ML endpoint is called after the upgrade
      */
     public void testSnapshotUpgrader() throws Exception {
-        hlrc = new HLRC(client()).machineLearning();
+        hlrc = HLRC(client()).machineLearning();
         Request adjustLoggingLevels = new Request("PUT", "/_cluster/settings");
         adjustLoggingLevels.setJsonEntity("{\"persistent\": {" + "\"logger.org.elasticsearch.xpack.ml\": \"trace\"" + "}}");
         client().performRequest(adjustLoggingLevels);