Improve ESRestTestCase when running with different node versions. (elastic#70361)

Backport of the testing related changes from elastic#70314:

Older versions don't support component / composable index templates
and/or data streams. Yet the test base class tries to remove objects
after each test, which adds a significant number of lines to the
log files (which slows the tests down). The ESRestTestCase will
now check whether all nodes have a specific version and then decide
whether data streams and component / composable index templates will
be deleted.

Also ensured that the logstash-index-template and security-index-template
aren't deleted between tests, these templates are builtin templates that
ES will install if missing. So if tests remove these templates between tests
then ES will add these template back almost immediately. These causes
many log lines and a lot of cluster state updates, which slow tests down.

Also removed old debug log config that was enabled to investigate
a build failure (elastic#46091), but has been closed. However the debug logging
added many lines log lines to the log files. Note this change wasn't part
of elastic#70314.

Relates to elastic#69973

Author: martijnvg

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1213,10 +1213,6 @@ private void createConfiguration() {
         // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
         baseConfig.put("discovery.initial_state_timeout", "0s");
 
-        // TODO: Remove these once https://github.com/elastic/elasticsearch/issues/46091 is fixed
-        baseConfig.put("logger.org.elasticsearch.action.support.master", "DEBUG");
-        baseConfig.put("logger.org.elasticsearch.cluster.coordination", "DEBUG");
-
         HashSet<String> overriden = new HashSet<>(baseConfig.keySet());
         overriden.retainAll(settings.keySet());
         overriden.removeAll(OVERRIDABLE_SETTINGS);

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -602,58 +602,63 @@ private void wipeCluster() throws Exception {
                  * slows down the test because xpack will just recreate
                  * them.
                  */
-                try {
-                    Request getTemplatesRequest = new Request("GET", "_index_template");
-                    getTemplatesRequest.setOptions(allowTypesRemovalWarnings());
-                    Map<String, Object> composableIndexTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent,
-                        EntityUtils.toString(adminClient().performRequest(getTemplatesRequest).getEntity()), false);
-                    List<String> names = ((List<?>) composableIndexTemplates.get("index_templates")).stream()
-                        .map(ct -> (String) ((Map<?, ?>) ct).get("name"))
-                        .filter(name -> isXPackTemplate(name) == false)
-                        .collect(Collectors.toList());
-                    // Ideally we would want to check the version of the elected master node and
-                    // send the delete request directly to that node.
-                    if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_13_0))) {
-                        try {
-                            adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names)));
-                        } catch (ResponseException e) {
-                            logger.debug(new ParameterizedMessage("unable to remove multiple composable index template {}", names), e);
-                        }
-                    } else {
-                        for (String name : names) {
+                // In case of bwc testing, if all nodes are before 7.7.0 then no need to attempt to delete component and composable
+                // index templates, because these were introduced in 7.7.0:
+                if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_7_0))) {
+                    try {
+                        Request getTemplatesRequest = new Request("GET", "_index_template");
+                        getTemplatesRequest.setOptions(allowTypesRemovalWarnings());
+                        Map<String, Object> composableIndexTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent,
+                            EntityUtils.toString(adminClient().performRequest(getTemplatesRequest).getEntity()), false);
+                        List<String> names = ((List<?>) composableIndexTemplates.get("index_templates")).stream()
+                            .map(ct -> (String) ((Map<?, ?>) ct).get("name"))
+                            .filter(name -> isXPackTemplate(name) == false)
+                            .collect(Collectors.toList());
+                        // Ideally we would want to check the version of the elected master node and
+                        // send the delete request directly to that node.
+                        if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_13_0))) {
                             try {
-                                adminClient().performRequest(new Request("DELETE", "_index_template/" + name));
+                                adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names)));
                             } catch (ResponseException e) {
-                                logger.debug(new ParameterizedMessage("unable to remove composable index template {}", name), e);
+                                logger.debug(new ParameterizedMessage("unable to remove multiple composable index template {}", names), e);
+                            }
+                        } else {
+                            for (String name : names) {
+                                try {
+                                    adminClient().performRequest(new Request("DELETE", "_index_template/" + name));
+                                } catch (ResponseException e) {
+                                    logger.debug(new ParameterizedMessage("unable to remove composable index template {}", name), e);
+                                }
                             }
                         }
+                    } catch (Exception e) {
+                        logger.debug("ignoring exception removing all composable index templates", e);
+                        // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
                     }
-                } catch (Exception e) {
-                    logger.debug("ignoring exception removing all composable index templates", e);
-                    // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
-                }
-                try {
-                    Request compReq = new Request("GET", "_component_template");
-                    compReq.setOptions(allowTypesRemovalWarnings());
-                    String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity());
-                    Map<String, Object> cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false);
-                    List<String> names = ((List<?>) cTemplates.get("component_templates")).stream()
-                        .map(ct -> (String) ((Map<?, ?>) ct).get("name"))
-                        .collect(Collectors.toList());
-                    for (String componentTemplate : names) {
-                        try {
-                            if (isXPackTemplate(componentTemplate)) {
-                                continue;
+                    try {
+                        Request compReq = new Request("GET", "_component_template");
+                        compReq.setOptions(allowTypesRemovalWarnings());
+                        String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity());
+                        Map<String, Object> cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false);
+                        List<String> names = ((List<?>) cTemplates.get("component_templates")).stream()
+                            .map(ct -> (String) ((Map<?, ?>) ct).get("name"))
+                            .collect(Collectors.toList());
+                        for (String componentTemplate : names) {
+                            try {
+                                if (isXPackTemplate(componentTemplate)) {
+                                    continue;
+                                }
+                                adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate));
+                            } catch (ResponseException e) {
+                                logger.debug(new ParameterizedMessage("unable to remove component template {}", componentTemplate), e);
                             }
-                            adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate));
-                        } catch (ResponseException e) {
-                            logger.debug(new ParameterizedMessage("unable to remove component template {}", componentTemplate), e);
                         }
+                    } catch (Exception e) {
+                        logger.debug("ignoring exception removing all component templates", e);
+                        // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
                     }
-                } catch (Exception e) {
-                    logger.debug("ignoring exception removing all component templates", e);
-                    // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
                 }
+                // Always check for legacy templates:
                 Request getLegacyTemplatesRequest = new Request("GET", "_template");
                 getLegacyTemplatesRequest.setOptions(allowTypesRemovalWarnings());
                 Map<String, Object> legacyTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent,
@@ -730,7 +735,7 @@ protected static void wipeAllIndices() throws IOException {
 
     protected static void wipeDataStreams() throws IOException {
         try {
-            if (hasXPack()) {
+            if (hasXPack() && nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_9_0))) {
                 adminClient().performRequest(new Request("DELETE", "_data_stream/*?expand_wildcards=all"));
             }
         } catch (ResponseException e) {
@@ -1442,6 +1447,8 @@ protected static boolean isXPackTemplate(String name) {
             case "synthetics-mappings":
             case ".snapshot-blob-cache":
             case ".deprecation-indexing-template":
+            case "logstash-index-template":
+            case "security-index-template":
                 return true;
             default:
                 return false;