From 0ee12725232a0a4381939bad9508a3915e7201f8 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 1 Feb 2019 16:08:27 -0700 Subject: [PATCH] Run Node deprecation checks locally (#38065) At times, we need to check for usage of deprecated settings in settings which should not be returned by the NodeInfo API. This commit changes the deprecation info API to run all node checks locally so that these settings can be checked without exposing them via any externally accessible API. --- client/rest-high-level/build.gradle | 2 +- .../deprecation/DeprecationInfoAction.java | 44 ++-- .../NodesDeprecationCheckAction.java | 127 ++++++++++++ .../NodesDeprecationCheckRequest.java | 50 +++++ .../NodesDeprecationCheckResponse.java | 53 +++++ .../DeprecationInfoActionResponseTests.java | 32 ++- .../NodesDeprecationCheckRequestTests.java | 33 +++ .../NodesDeprecationCheckResponseTests.java | 84 ++++++++ .../xpack/deprecation/Deprecation.java | 7 +- .../xpack/deprecation/DeprecationChecks.java | 6 +- .../deprecation/NodeDeprecationChecks.java | 190 ++++++------------ .../TransportDeprecationInfoAction.java | 62 +++--- .../TransportNodeDeprecationCheckAction.java | 76 +++++++ .../NodeDeprecationChecksTests.java | 61 +++--- 14 files changed, 600 insertions(+), 227 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 50a882c35774d..22e6252892a7d 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -105,7 +105,7 @@ integTestCluster { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API - setting 'xpack.ssl.certificate_authorities', 'testnode.crt' + setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt' setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks' setting 'indices.lifecycle.poll_interval', '1000ms' keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode' diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java index 111010e184a4a..8cb68a6429f11 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeReadRequest; @@ -35,7 +34,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -63,6 +61,23 @@ public static List filterChecks(List checks, Function mergeNodeIssues(NodesDeprecationCheckResponse response) { + Map> issueListMap = new HashMap<>(); + for (NodesDeprecationCheckAction.NodeResponse resp : response.getNodes()) { + for (DeprecationIssue issue : resp.getDeprecationIssues()) { + issueListMap.computeIfAbsent(issue, (key) -> new ArrayList<>()).add(resp.getNode().getName()); + } + } + + return issueListMap.entrySet().stream() + .map(entry -> { + DeprecationIssue issue = entry.getKey(); + String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; + return new DeprecationIssue(issue.getLevel(), issue.getMessage(), issue.getUrl(), + details + "(nodes impacted: " + entry.getValue() + ")"); + }).collect(Collectors.toList()); + } + @Override public RequestBuilder newRequestBuilder(ElasticsearchClient client) { return new RequestBuilder(client, this); @@ -167,32 +182,29 @@ public int hashCode() { * this function will run through all the checks and build out the final list of issues that exist in the * cluster. * - * @param nodesInfo The list of {@link NodeInfo} metadata objects for retrieving node-level information - * @param nodesStats The list of {@link NodeStats} metadata objects for retrieving node-level information * @param state The cluster state * @param indexNameExpressionResolver Used to resolve indices into their concrete names * @param indices The list of index expressions to evaluate using `indexNameExpressionResolver` * @param indicesOptions The options to use when resolving and filtering which indices to check * @param datafeeds The ml datafeed configurations - * @param clusterSettingsChecks The list of cluster-level checks - * @param nodeSettingsChecks The list of node-level checks + * @param nodeDeprecationResponse The response containing the deprecation issues found on each node * @param indexSettingsChecks The list of index-level checks that will be run across all specified * concrete indices + * @param clusterSettingsChecks The list of cluster-level checks * @param mlSettingsCheck The list of ml checks * @return The list of deprecation issues found in the cluster */ - public static DeprecationInfoAction.Response from(List nodesInfo, List nodesStats, ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - String[] indices, IndicesOptions indicesOptions, - List datafeeds, - List>clusterSettingsChecks, - List, List, DeprecationIssue>> nodeSettingsChecks, - List> indexSettingsChecks, - List> mlSettingsCheck) { + public static DeprecationInfoAction.Response from(ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver, + String[] indices, IndicesOptions indicesOptions, + List datafeeds, + NodesDeprecationCheckResponse nodeDeprecationResponse, + List> indexSettingsChecks, + List> clusterSettingsChecks, + List> mlSettingsCheck) { List clusterSettingsIssues = filterChecks(clusterSettingsChecks, (c) -> c.apply(state)); - List nodeSettingsIssues = filterChecks(nodeSettingsChecks, - (c) -> c.apply(nodesInfo, nodesStats)); + List nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse); List mlSettingsIssues = new ArrayList<>(); for (DatafeedConfig config : datafeeds) { mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config))); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java new file mode 100644 index 0000000000000..f95f3e4e7bc6c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.support.nodes.BaseNodeRequest; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings + * can be accessed in the deprecation checks. + */ +public class NodesDeprecationCheckAction extends Action { + public static final NodesDeprecationCheckAction INSTANCE = new NodesDeprecationCheckAction(); + public static final String NAME = "cluster:admin/xpack/deprecation/nodes/info"; + + private NodesDeprecationCheckAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, INSTANCE, new NodesDeprecationCheckRequest()); + } + + @Override + public NodesDeprecationCheckResponse newResponse() { + return new NodesDeprecationCheckResponse(); + } + + public static class NodeRequest extends BaseNodeRequest { + + NodesDeprecationCheckRequest request; + + public NodeRequest() {} + public NodeRequest(String nodeId, NodesDeprecationCheckRequest request) { + super(nodeId); + this.request = request; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + request = new NodesDeprecationCheckRequest(); + request.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + request.writeTo(out); + } + } + + public static class NodeResponse extends BaseNodeResponse { + private List deprecationIssues; + + public NodeResponse() { + super(); + } + + public NodeResponse(DiscoveryNode node, List deprecationIssues) { + super(node); + this.deprecationIssues = deprecationIssues; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + deprecationIssues = in.readList(DeprecationIssue::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(this.deprecationIssues); + } + + public static NodeResponse readNodeResponse(StreamInput in) throws IOException { + NodeResponse nodeResponse = new NodeResponse(); + nodeResponse.readFrom(in); + return nodeResponse; + } + + public List getDeprecationIssues() { + return deprecationIssues; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeResponse that = (NodeResponse) o; + return Objects.equals(getDeprecationIssues(), that.getDeprecationIssues()) + && Objects.equals(getNode(), that.getNode()); + } + + @Override + public int hashCode() { + return Objects.hash(getNode(), getDeprecationIssues()); + } + } + + public static class RequestBuilder extends NodesOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, + Action action, + NodesDeprecationCheckRequest request) { + super(client, action, request); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java new file mode 100644 index 0000000000000..af7b2da6f55eb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +public class NodesDeprecationCheckRequest extends BaseNodesRequest { + public NodesDeprecationCheckRequest() {} + + public NodesDeprecationCheckRequest(String... nodesIds) { + super(nodesIds); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash((Object[]) this.nodesIds()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + NodesDeprecationCheckRequest that = (NodesDeprecationCheckRequest) obj; + return Arrays.equals(this.nodesIds(), that.nodesIds()); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java new file mode 100644 index 0000000000000..db7dbc6a381e2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class NodesDeprecationCheckResponse extends BaseNodesResponse { + + public NodesDeprecationCheckResponse() {} + + public NodesDeprecationCheckResponse(ClusterName clusterName, + List nodes, + List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodesDeprecationCheckAction.NodeResponse::readNodeResponse); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeStreamableList(nodes); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodesDeprecationCheckResponse that = (NodesDeprecationCheckResponse) o; + return Objects.equals(getClusterName(), that.getClusterName()) + && Objects.equals(getNodes(), that.getNodes()) + && Objects.equals(failures(), that.failures()); + } + + @Override + public int hashCode() { + return Objects.hash(getClusterName(), getNodes(), failures()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java index 21481df3dc3d3..89992fee1506c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java @@ -5,10 +5,7 @@ */ package org.elasticsearch.xpack.core.deprecation; -import org.elasticsearch.Build; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -31,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -78,12 +74,6 @@ public void testFrom() throws IOException { DiscoveryNode discoveryNode = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9300), "test"); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(metadata).build(); - List nodeInfos = Collections.singletonList(new NodeInfo(Version.CURRENT, Build.CURRENT, - discoveryNode, null, null, null, null, - null, null, null, null, null, null)); - List nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null, - null, null, null, null, null, null, null, null, - null, null, null, null)); List datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo")); IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY); IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, @@ -97,11 +87,6 @@ public void testFrom() throws IOException { Collections.unmodifiableList(Arrays.asList( (s) -> clusterIssueFound ? foundIssue : null )); - List, List, DeprecationIssue>> nodeSettingsChecks = - Collections.unmodifiableList(Arrays.asList( - (ln, ls) -> nodeIssueFound ? foundIssue : null - )); - List> indexSettingsChecks = Collections.unmodifiableList(Arrays.asList( (idx) -> indexIssueFound ? foundIssue : null @@ -111,9 +96,17 @@ public void testFrom() throws IOException { (idx) -> mlIssueFound ? foundIssue : null )); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state, + NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( + new ClusterName(randomAlphaOfLength(5)), + nodeIssueFound + ? Collections.singletonList( + new NodesDeprecationCheckAction.NodeResponse(discoveryNode, Collections.singletonList(foundIssue))) + : Collections.emptyList(), + Collections.emptyList()); + + DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(state, resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds, - clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks); + nodeDeprecationIssues, indexSettingsChecks, clusterSettingsChecks, mlSettingsChecks); if (clusterIssueFound) { assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); @@ -122,7 +115,10 @@ public void testFrom() throws IOException { } if (nodeIssueFound) { - assertThat(response.getNodeSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); + String details = foundIssue.getDetails() != null ? foundIssue.getDetails() + " " : ""; + DeprecationIssue mergedFoundIssue = new DeprecationIssue(foundIssue.getLevel(), foundIssue.getMessage(), foundIssue.getUrl(), + details + "(nodes impacted: [" + discoveryNode.getName() + "])"); + assertThat(response.getNodeSettingsIssues(), equalTo(Collections.singletonList(mergedFoundIssue))); } else { assertTrue(response.getNodeSettingsIssues().isEmpty()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java new file mode 100644 index 0000000000000..8dd7255a7f15a --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.io.IOException; + +public class NodesDeprecationCheckRequestTests + extends AbstractStreamableTestCase { + + @Override + protected NodesDeprecationCheckRequest createBlankInstance() { + return new NodesDeprecationCheckRequest(); + } + + @Override + protected NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) throws IOException { + int newSize = randomValueOtherThan(instance.nodesIds().length, () -> randomIntBetween(0,10)); + String[] newNodeIds = randomArray(newSize, newSize, String[]::new, () -> randomAlphaOfLengthBetween(5, 10)); + return new NodesDeprecationCheckRequest(newNodeIds); + } + + @Override + protected NodesDeprecationCheckRequest createTestInstance() { + return new NodesDeprecationCheckRequest(randomArray(0, 10, String[]::new, + ()-> randomAlphaOfLengthBetween(5,10))); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java new file mode 100644 index 0000000000000..143c0e2f5ad50 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.Version; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class NodesDeprecationCheckResponseTests + extends AbstractStreamableTestCase { + + @Override + protected NodesDeprecationCheckResponse createBlankInstance() { + return new NodesDeprecationCheckResponse(); + } + + @Override + protected NodesDeprecationCheckResponse createTestInstance() { + + List responses = + Arrays.asList(randomArray(1, 10, NodesDeprecationCheckAction.NodeResponse[]::new, + NodesDeprecationCheckResponseTests::randomNodeResponse)); + return new NodesDeprecationCheckResponse(new ClusterName(randomAlphaOfLength(10)), + responses, + Collections.emptyList()); + } + + @Override + protected NodesDeprecationCheckResponse mutateInstance(NodesDeprecationCheckResponse instance) throws IOException { + int mutate = randomIntBetween(1,3); + switch (mutate) { + case 1: + List responses = new ArrayList<>(instance.getNodes()); + responses.add(randomNodeResponse()); + return new NodesDeprecationCheckResponse(instance.getClusterName(), responses, instance.failures()); + case 2: + ArrayList failures = new ArrayList<>(instance.failures()); + failures.add(new FailedNodeException("test node", "test failure", new RuntimeException(randomAlphaOfLength(10)))); + return new NodesDeprecationCheckResponse(instance.getClusterName(), instance.getNodes(), failures); + case 3: + String clusterName = randomValueOtherThan(instance.getClusterName().value(), () -> randomAlphaOfLengthBetween(5,15)); + return new NodesDeprecationCheckResponse(new ClusterName(clusterName), instance.getNodes(), instance.failures()); + default: + fail("invalid mutation"); + } + + return super.mutateInstance(instance); + } + + private static DiscoveryNode randomDiscoveryNode() throws Exception { + InetAddress inetAddress = InetAddress.getByAddress(randomAlphaOfLength(5), + new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1}); + TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); + + return new DiscoveryNode(randomAlphaOfLength(5), randomAlphaOfLength(5), transportAddress, + Collections.emptyMap(), Collections.emptySet(), Version.CURRENT); + } + + private static NodesDeprecationCheckAction.NodeResponse randomNodeResponse() { + DiscoveryNode node; + try { + node = randomDiscoveryNode(); + } catch (Exception e) { + throw new RuntimeException(e); + } + List issuesList = Arrays.asList(randomArray(0,10, DeprecationIssue[]::new, + DeprecationIssueTests::createTestInstance)); + return new NodesDeprecationCheckAction.NodeResponse(node, issuesList); + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index cede3eb309151..9bfbe352f839a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -19,7 +19,9 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -30,7 +32,10 @@ public class Deprecation extends Plugin implements ActionPlugin { @Override public List> getActions() { - return Collections.singletonList(new ActionHandler<>(DeprecationInfoAction.INSTANCE, TransportDeprecationInfoAction.class)); + return Collections.unmodifiableList(Arrays.asList( + new ActionHandler<>(DeprecationInfoAction.INSTANCE, TransportDeprecationInfoAction.class), + new ActionHandler<>(NodesDeprecationCheckAction.INSTANCE, TransportNodeDeprecationCheckAction.class) + )); } @Override diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index fa8a5cfbcd985..6a22f694771dc 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -5,10 +5,10 @@ */ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -37,7 +37,7 @@ private DeprecationChecks() { ClusterDeprecationChecks::checkClusterName )); - static List, List, DeprecationIssue>> NODE_SETTINGS_CHECKS = + static List> NODE_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( NodeDeprecationChecks::httpEnabledSettingRemoved, NodeDeprecationChecks::auditLogPrefixSettingsCheck, diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index 8cb1424d4098f..c88345466962a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -6,15 +6,12 @@ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; -import java.util.List; -import java.util.stream.Collectors; - import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_TYPE_SETTING; import static org.elasticsearch.discovery.zen.SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING; @@ -24,230 +21,171 @@ */ public class NodeDeprecationChecks { - static DeprecationIssue httpEnabledSettingRemoved(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().hasValue(NetworkModule.HTTP_ENABLED.getKey())) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue httpEnabledSettingRemoved(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.hasValue(NetworkModule.HTTP_ENABLED.getKey())) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "HTTP Enabled setting removed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#remove-http-enabled", - "nodes with http.enabled set: " + nodesFound); + "the HTTP Enabled setting has been removed"); } return null; } - static DeprecationIssue auditLogPrefixSettingsCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().getByPrefix("xpack.security.audit.logfile.prefix").isEmpty() == false) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue auditLogPrefixSettingsCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getByPrefix("xpack.security.audit.logfile.prefix").isEmpty() == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Audit log node info settings renamed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#audit-logfile-local-node-info", - "nodes with audit log settings that have been renamed: " + nodesFound); + "the audit log is now structured JSON"); } return null; } - static DeprecationIssue auditIndexSettingsCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> (nodeInfo.getSettings().getByPrefix("xpack.security.audit.outputs").isEmpty() == false) - || (nodeInfo.getSettings().getByPrefix("xpack.security.audit.index").isEmpty() == false)) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue auditIndexSettingsCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getByPrefix("xpack.security.audit.outputs").isEmpty() == false + || nodeSettings.getByPrefix("xpack.security.audit.index").isEmpty() == false) { + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Audit index output type removed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#remove-audit-index-output", - "nodes with audit index output type settings: " + nodesFound); + "recommended replacement is the logfile audit output type"); } return null; } - static DeprecationIssue indexThreadPoolCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().getByPrefix("thread_pool.index.").isEmpty() == false) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue indexThreadPoolCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getByPrefix("thread_pool.index.").isEmpty() == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Index thread pool removed in favor of combined write thread pool", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_index_thread_pool", - "nodes with index thread pool settings: " + nodesFound); + "the write threadpool is now used for all writes"); } return null; } - static DeprecationIssue bulkThreadPoolCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().getByPrefix("thread_pool.bulk.").isEmpty() == false) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + + static DeprecationIssue bulkThreadPoolCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getByPrefix("thread_pool.bulk.").isEmpty() == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Bulk thread pool renamed to write thread pool", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#write-thread-pool-fallback", - "nodes with bulk thread pool settings: " + nodesFound); + "the write threadpool is now used for all writes"); } return null; } - static DeprecationIssue tribeNodeCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().getByPrefix("tribe.").isEmpty() == false) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue tribeNodeCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getByPrefix("tribe.").isEmpty() == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Tribe Node removed in favor of Cross Cluster Search", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_tribe_node_removed", - "nodes with tribe node settings: " + nodesFound); + "tribe node functionality has been removed in favor of cross-cluster search"); } return null; } - static DeprecationIssue authRealmsTypeCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().getGroups("xpack.security.authc.realms").size() > 0) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - - if (nodesFound.size() > 0) { + static DeprecationIssue authRealmsTypeCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getGroups("xpack.security.authc.realms").size() > 0) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Security realm settings structure changed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#include-realm-type-in-setting", - "nodes have authentication realm configuration which must be updated at time of upgrade to 7.0: " + nodesFound); + "these settings must be updated to the new format while the node is offline during the upgrade to 7.0"); } return null; } - static DeprecationIssue httpPipeliningCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().hasValue(HttpTransportSettings.SETTING_PIPELINING.getKey())) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue httpPipeliningCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.hasValue(HttpTransportSettings.SETTING_PIPELINING.getKey())) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "HTTP pipelining setting removed as pipelining is now mandatory", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#remove-http-pipelining-setting", - "nodes with http.pipelining set: " + nodesFound); + ""); } return null; } - static DeprecationIssue discoveryConfigurationCheck(List nodeInfos, List nodeStats) { - - List nodesFound = nodeInfos.stream() - // These checks only apply in Zen2, which is the new default in 7.0 and can't be used in 6.x, so only apply the checks if this - // node does not have a discovery type explicitly set - .filter(nodeInfo -> nodeInfo.getSettings().hasValue(DISCOVERY_TYPE_SETTING.getKey()) == false) + static DeprecationIssue discoveryConfigurationCheck(Settings nodeSettings, PluginsAndModules plugins) { + // These checks only apply in Zen2, which is the new default in 7.0 and can't be used in 6.x, so only apply the checks if this + // node does not have a discovery type explicitly set + if (nodeSettings.hasValue(DISCOVERY_TYPE_SETTING.getKey()) == false // This only checks for `ping.unicast.hosts` and `hosts_provider` because `cluster.initial_master_nodes` does not exist in 6.x - .filter(nodeInfo -> nodeInfo.getSettings().hasValue(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) == false) - .filter(nodeInfo -> nodeInfo.getSettings().hasValue(DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()) == false) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + && nodeSettings.hasValue(DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.getKey()) == false + && nodeSettings.hasValue(DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()) == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Discovery configuration is required in production mode", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_discovery_configuration_is_required_in_production", - "nodes which do not have discovery configured: " + nodesFound); + ""); } return null; } - static DeprecationIssue watcherNotificationsSecureSettingsCheck(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream().filter(nodeInfo -> - (false == nodeInfo.getSettings().getByPrefix("xpack.notification.email.account.") - .filter(s -> s.endsWith(".smtp.password")).isEmpty()) - || (false == nodeInfo.getSettings().getByPrefix("xpack.notification.hipchat.account.") - .filter(s -> s.endsWith(".auth_token")).isEmpty()) - || (false == nodeInfo.getSettings().getByPrefix("xpack.notification.jira.account.") - .filter(s -> s.endsWith(".url") || s.endsWith(".user") || s.endsWith(".password")).isEmpty()) - || (false == nodeInfo.getSettings().getByPrefix("xpack.notification.pagerduty.account.") - .filter(s -> s.endsWith(".service_api_key")).isEmpty()) - || (false == nodeInfo.getSettings().getByPrefix("xpack.notification.slack.account.").filter(s -> s.endsWith(".url")) - .isEmpty())) - .map(nodeInfo -> nodeInfo.getNode().getName()).collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue watcherNotificationsSecureSettingsCheck(Settings nodeSettings, PluginsAndModules plugins) { + if (false == nodeSettings.getByPrefix("xpack.notification.email.account.").filter(s -> s.endsWith(".smtp.password")).isEmpty() + || false == nodeSettings.getByPrefix("xpack.notification.hipchat.account.").filter(s -> s.endsWith(".auth_token")).isEmpty() + || false == nodeSettings.getByPrefix("xpack.notification.jira.account.") + .filter(s -> s.endsWith(".url") || s.endsWith(".user") || s.endsWith(".password")).isEmpty() + || false == nodeSettings.getByPrefix("xpack.notification.pagerduty.account.") + .filter(s -> s.endsWith(".service_api_key")).isEmpty() + || false == nodeSettings.getByPrefix("xpack.notification.slack.account.").filter(s -> s.endsWith(".url")).isEmpty()) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, - "Watcher notification accounts' authentication settings must be defined securely", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + - "#watcher-notifications-account-settings", - "nodes which have insecure notification account settings are: " + nodesFound); + "Watcher notification accounts' authentication settings must be defined securely", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "#watcher-notifications-account-settings", + "account authentication settings must use the keystore"); } return null; } - static DeprecationIssue azureRepositoryChanges(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> - nodeInfo.getPlugins().getPluginInfos().stream() - .anyMatch(pluginInfo -> "repository-azure".equals(pluginInfo.getName())) - ).map(nodeInfo -> nodeInfo.getNode().getName()).collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue azureRepositoryChanges(Settings nodeSettings, PluginsAndModules plugins) { + if (plugins.getPluginInfos().stream().anyMatch(pluginInfo -> "repository-azure".equals(pluginInfo.getName()))) { return new DeprecationIssue(DeprecationIssue.Level.WARNING, "Azure Repository settings changed", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_azure_repository_plugin", - "nodes with repository-azure installed: " + nodesFound); + "see breaking changes list for details"); } return null; } - static DeprecationIssue gcsRepositoryChanges(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> - nodeInfo.getPlugins().getPluginInfos().stream() - .anyMatch(pluginInfo -> "repository-gcs".equals(pluginInfo.getName())) - ).map(nodeInfo -> nodeInfo.getNode().getName()).collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue gcsRepositoryChanges(Settings nodeSettings, PluginsAndModules plugins) { + if (plugins.getPluginInfos().stream().anyMatch(pluginInfo -> "repository-gcs".equals(pluginInfo.getName()))) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, "GCS Repository settings changed", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_google_cloud_storage_repository_plugin", - "nodes with repository-gcs installed: " + nodesFound); + "see breaking changes list for details"); } return null; } - static DeprecationIssue fileDiscoveryPluginRemoved(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> - nodeInfo.getPlugins().getPluginInfos().stream() - .anyMatch(pluginInfo -> "discovery-file".equals(pluginInfo.getName())) - ).map(nodeInfo -> nodeInfo.getNode().getName()).collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue fileDiscoveryPluginRemoved(Settings nodeSettings, PluginsAndModules plugins) { + if (plugins.getPluginInfos().stream().anyMatch(pluginInfo -> "discovery-file".equals(pluginInfo.getName()))) { return new DeprecationIssue(DeprecationIssue.Level.WARNING, "File-based discovery is no longer a plugin and uses a different path", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_file_based_discovery_plugin", - "nodes with discovery-file installed: " + nodesFound); + "the location of the hosts file used for file-based discovery has changed to $ES_PATH_CONF/unicast_hosts.txt " + + "(from $ES_PATH_CONF/file-discovery/unicast_hosts.txt)"); } return null; } - static DeprecationIssue defaultSSLSettingsRemoved(List nodeInfos, List nodeStats) { - List nodesFound = nodeInfos.stream() - .filter(nodeInfo -> nodeInfo.getSettings().getByPrefix("xpack.ssl").isEmpty() == false) - .map(nodeInfo -> nodeInfo.getNode().getName()) - .collect(Collectors.toList()); - if (nodesFound.size() > 0) { + static DeprecationIssue defaultSSLSettingsRemoved(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.getByPrefix("xpack.ssl").isEmpty() == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Default TLS/SSL settings have been removed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#tls-setting-fallback", - "Nodes with default TLS/SSL settings: " + nodesFound); + "each component must have TLS/SSL configured explicitly"); } return null; } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index ef2dca1180b7c..dd700bc6423d6 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -6,10 +6,7 @@ package org.elasticsearch.xpack.deprecation; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.client.node.NodeClient; @@ -20,7 +17,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.threadpool.ThreadPool; @@ -28,17 +24,19 @@ import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckRequest; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS; -import static org.elasticsearch.xpack.deprecation.DeprecationChecks.NODE_SETTINGS_CHECKS; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.ML_SETTINGS_CHECKS; public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction listener) { if (licenseState.isDeprecationAllowed()) { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest("_local").settings(true).plugins(true); - NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("_local").fs(true); + NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); + executeAsyncWithOrigin(client, DEPRECATION_ORIGIN, NodesDeprecationCheckAction.INSTANCE, nodeDepReq, + ActionListener.wrap(response -> { + if (response.hasFailures()) { + List failedNodeIds = response.failures().stream() + .map(failure -> failure.nodeId() + ": " + failure.getMessage()) + .collect(Collectors.toList()); + logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); + for (FailedNodeException failure : response.failures()) { + logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); + } + } + getDatafeedConfigs(ActionListener.wrap( + datafeeds -> { + listener.onResponse( + DeprecationInfoAction.Response.from(state, indexNameExpressionResolver, + request.indices(), request.indicesOptions(), datafeeds, + response, INDEX_SETTINGS_CHECKS, CLUSTER_SETTINGS_CHECKS, + ML_SETTINGS_CHECKS)); + }, + listener::onFailure + )); - final ThreadContext threadContext = client.threadPool().getThreadContext(); - executeAsyncWithOrigin(threadContext, DEPRECATION_ORIGIN, nodesInfoRequest, - ActionListener.wrap( - nodesInfoResponse -> { - if (nodesInfoResponse.hasFailures()) { - throw nodesInfoResponse.failures().get(0); - } - executeAsyncWithOrigin(threadContext, DEPRECATION_ORIGIN, nodesStatsRequest, - ActionListener.wrap( - nodesStatsResponse -> { - if (nodesStatsResponse.hasFailures()) { - throw nodesStatsResponse.failures().get(0); - } - getDatafeedConfigs(ActionListener.wrap( - datafeeds -> { - listener.onResponse( - DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(), - nodesStatsResponse.getNodes(), state, indexNameExpressionResolver, - request.indices(), request.indicesOptions(), datafeeds, - CLUSTER_SETTINGS_CHECKS, NODE_SETTINGS_CHECKS, - INDEX_SETTINGS_CHECKS, ML_SETTINGS_CHECKS)); - }, - listener::onFailure - )); - }, listener::onFailure), - client.admin().cluster()::nodesStats); - }, listener::onFailure), client.admin().cluster()::nodesInfo); + }, listener::onFailure)); } else { listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION)); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java new file mode 100644 index 0000000000000..8534f15c11d13 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckRequest; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckResponse; + +import java.util.List; + +public class TransportNodeDeprecationCheckAction extends TransportNodesAction { + + private final Settings settings; + private final PluginsService pluginsService; + + @Inject + public TransportNodeDeprecationCheckAction(Settings settings, ThreadPool threadPool, + ClusterService clusterService, TransportService transportService, + PluginsService pluginsService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, NodesDeprecationCheckAction.NAME, threadPool, clusterService, transportService, actionFilters, + indexNameExpressionResolver, + NodesDeprecationCheckRequest::new, + NodesDeprecationCheckAction.NodeRequest::new, + ThreadPool.Names.GENERIC, + NodesDeprecationCheckAction.NodeResponse.class); + this.settings = settings; + this.pluginsService = pluginsService; + } + + @Override + protected NodesDeprecationCheckResponse newResponse(NodesDeprecationCheckRequest request, + List nodeResponses, + List failures) { + return new NodesDeprecationCheckResponse(clusterService.getClusterName(), nodeResponses, failures); + } + + @Override + protected NodesDeprecationCheckAction.NodeRequest newNodeRequest(String nodeId, NodesDeprecationCheckRequest request) { + return new NodesDeprecationCheckAction.NodeRequest(nodeId, request); + } + + @Override + protected NodesDeprecationCheckAction.NodeResponse newNodeResponse() { + return new NodesDeprecationCheckAction.NodeResponse(); + } + + @Override + protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request) { + List issues = DeprecationInfoAction.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS, + (c) -> c.apply(settings, pluginsService.info())); + + return new NodesDeprecationCheckAction.NodeResponse(transportService.getLocalNode(), issues); + } + + +} diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 3bdcb9a2ded53..915edbbfb71bc 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -61,7 +61,7 @@ private void assertSettingsAndIssue(String key, String value, DeprecationIssue e List nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null, null, null, null, null, new FsInfo(0L, null, paths), null, null, null, null, null, null, null)); - List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(nodeInfos, nodeStats)); + List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(settings, pluginsAndModules)); assertEquals(singletonList(expected), issues); } @@ -70,7 +70,7 @@ public void testHttpEnabledCheck() { "HTTP Enabled setting removed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#remove-http-enabled", - "nodes with http.enabled set: [node_check]"); + "the HTTP Enabled setting has been removed"); assertSettingsAndIssue("http.enabled", Boolean.toString(randomBoolean()), expected); } @@ -79,20 +79,22 @@ public void testAuditLoggingPrefixSettingsCheck() { "Audit log node info settings renamed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#audit-logfile-local-node-info", - "nodes with audit log settings that have been renamed: [node_check]"); - assertSettingsAndIssue("xpack.security.audit.logfile.prefix.emit_node_host_address", Boolean.toString(randomBoolean()), expected); + "the audit log is now structured JSON"); + assertSettingsAndIssue("xpack.security.audit.logfile.prefix.emit_node_host_address", + Boolean.toString(randomBoolean()), expected); assertSettingsAndIssue("xpack.security.audit.logfile.prefix.emit_node_host_name", Boolean.toString(randomBoolean()), expected); assertSettingsAndIssue("xpack.security.audit.logfile.prefix.emit_node_name", Boolean.toString(randomBoolean()), expected); } public void testAuditIndexSettingsCheck() { DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Audit index output type removed", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#remove-audit-index-output", - "nodes with audit index output type settings: [node_check]"); + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "#remove-audit-index-output", + "recommended replacement is the logfile audit output type"); assertSettingsAndIssue("xpack.security.audit.outputs", randomFrom("[index]", "[\"index\", \"logfile\"]"), expected); assertSettingsAndIssue("xpack.security.audit.index.events.emit_request_body", Boolean.toString(randomBoolean()), expected); - assertSettingsAndIssue("xpack.security.audit.index.client.xpack.security.transport.ssl.enabled", Boolean.toString(randomBoolean()), - expected); + assertSettingsAndIssue("xpack.security.audit.index.client.xpack.security.transport.ssl.enabled", + Boolean.toString(randomBoolean()), expected); assertSettingsAndIssue("xpack.security.audit.index.client.cluster.name", randomAlphaOfLength(4), expected); assertSettingsAndIssue("xpack.security.audit.index.settings.index.number_of_shards", Integer.toString(randomInt()), expected); assertSettingsAndIssue("xpack.security.audit.index.events.include", @@ -106,7 +108,7 @@ public void testIndexThreadPoolCheck() { "Index thread pool removed in favor of combined write thread pool", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_index_thread_pool", - "nodes with index thread pool settings: [node_check]"); + "the write threadpool is now used for all writes"); assertSettingsAndIssue("thread_pool.index.size", Integer.toString(randomIntBetween(1, 20000)), expected); assertSettingsAndIssue("thread_pool.index.queue_size", Integer.toString(randomIntBetween(1, 20000)), expected); } @@ -116,7 +118,7 @@ public void testBulkThreadPoolCheck() { "Bulk thread pool renamed to write thread pool", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#write-thread-pool-fallback", - "nodes with bulk thread pool settings: [node_check]"); + "the write threadpool is now used for all writes"); assertSettingsAndIssue("thread_pool.bulk.size", Integer.toString(randomIntBetween(1, 20000)), expected); assertSettingsAndIssue("thread_pool.bulk.queue_size", Integer.toString(randomIntBetween(1, 20000)), expected); } @@ -126,36 +128,37 @@ public void testWatcherNotificationsSecureSettings() { "Watcher notification accounts' authentication settings must be defined securely", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#watcher-notifications-account-settings", - "nodes which have insecure notification account settings are: [node_check]"); + "account authentication settings must use the keystore"); assertSettingsAndIssue("xpack.notification.email.account." + randomAlphaOfLength(4) + ".smtp.password", randomAlphaOfLength(4), expected); expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Watcher notification accounts' authentication settings must be defined securely", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#watcher-notifications-account-settings", - "nodes which have insecure notification account settings are: [node_check]"); + "account authentication settings must use the keystore"); assertSettingsAndIssue("xpack.notification.hipchat.account." + randomAlphaOfLength(4) + ".auth_token", randomAlphaOfLength(4), expected); expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Watcher notification accounts' authentication settings must be defined securely", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#watcher-notifications-account-settings", - "nodes which have insecure notification account settings are: [node_check]"); + "account authentication settings must use the keystore"); assertSettingsAndIssue("xpack.notification.jira.account." + randomAlphaOfLength(4) + ".url", randomAlphaOfLength(4), expected); assertSettingsAndIssue("xpack.notification.jira.account." + randomAlphaOfLength(4) + ".user", randomAlphaOfLength(4), expected); - assertSettingsAndIssue("xpack.notification.jira.account." + randomAlphaOfLength(4) + ".password", randomAlphaOfLength(4), expected); + assertSettingsAndIssue("xpack.notification.jira.account." + randomAlphaOfLength(4) + ".password", + randomAlphaOfLength(4), expected); expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Watcher notification accounts' authentication settings must be defined securely", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#watcher-notifications-account-settings", - "nodes which have insecure notification account settings are: [node_check]"); + "account authentication settings must use the keystore"); assertSettingsAndIssue("xpack.notification.pagerduty.account." + randomAlphaOfLength(4) + ".service_api_key", randomAlphaOfLength(4), expected); expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Watcher notification accounts' authentication settings must be defined securely", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#watcher-notifications-account-settings", - "nodes which have insecure notification account settings are: [node_check]"); + "account authentication settings must use the keystore"); assertSettingsAndIssue("xpack.notification.slack.account." + randomAlphaOfLength(4) + ".url", randomAlphaOfLength(4), expected); } @@ -165,7 +168,7 @@ public void testTribeNodeCheck() { "Tribe Node removed in favor of Cross Cluster Search", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_tribe_node_removed", - "nodes with tribe node settings: [node_check]"); + "tribe node functionality has been removed in favor of cross-cluster search"); assertSettingsAndIssue(tribeSetting, randomAlphaOfLength(5), expected); } @@ -176,7 +179,7 @@ public void testAuthenticationRealmTypeCheck() { "Security realm settings structure changed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#include-realm-type-in-setting", - "nodes have authentication realm configuration which must be updated at time of upgrade to 7.0: [node_check]"); + "these settings must be updated to the new format while the node is offline during the upgrade to 7.0"); assertSettingsAndIssue(authRealmType, randomAlphaOfLength(5), expected); } @@ -185,7 +188,7 @@ public void testHttpPipeliningCheck() { "HTTP pipelining setting removed as pipelining is now mandatory", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#remove-http-pipelining-setting", - "nodes with http.pipelining set: [node_check]"); + ""); assertSettingsAndIssue("http.pipelining", Boolean.toString(randomBoolean()), expected); } @@ -206,7 +209,8 @@ null, null, null, null, new FsInfo(0L, null, paths), null, null, null, discoveryNode, hostsProviderSettings, osInfo, null, null, null, null, null, pluginsAndModules, null, null)); - List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(nodeInfos, nodeStats)); + List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, + c -> c.apply(hostsProviderSettings, pluginsAndModules)); assertTrue(issues.isEmpty()); } @@ -218,7 +222,8 @@ null, null, null, null, new FsInfo(0L, null, paths), null, null, null, discoveryNode, hostsProviderSettings, osInfo, null, null, null, null, null, pluginsAndModules, null, null)); - List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(nodeInfos, nodeStats)); + List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, + c -> c.apply(hostsProviderSettings, pluginsAndModules)); assertTrue(issues.isEmpty()); } @@ -233,8 +238,9 @@ null, null, null, null, new FsInfo(0L, null, paths), null, null, null, "Discovery configuration is required in production mode", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_discovery_configuration_is_required_in_production", - "nodes which do not have discovery configured: [node_check]"); - List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(nodeInfos, nodeStats)); + ""); + List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, + c -> c.apply(hostsProviderSettings, pluginsAndModules)); assertEquals(singletonList(expected), issues); } @@ -251,7 +257,7 @@ public void testAzurePluginCheck() { "Azure Repository settings changed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_azure_repository_plugin", - "nodes with repository-azure installed: [node_check]"); + "see breaking changes list for details"); assertSettingsAndIssue("foo", "bar", expected); } @@ -266,7 +272,7 @@ public void testGCSPluginCheck() { "GCS Repository settings changed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_google_cloud_storage_repository_plugin", - "nodes with repository-gcs installed: [node_check]"); + "see breaking changes list for details"); assertSettingsAndIssue("foo", "bar", expected); } @@ -281,7 +287,8 @@ public void testFileDiscoveryPluginCheck() { "File-based discovery is no longer a plugin and uses a different path", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#_file_based_discovery_plugin", - "nodes with discovery-file installed: [node_check]"); + "the location of the hosts file used for file-based discovery has changed to $ES_PATH_CONF/unicast_hosts.txt " + + "(from $ES_PATH_CONF/file-discovery/unicast_hosts.txt)"); assertSettingsAndIssue("foo", "bar", expected); } @@ -290,7 +297,7 @@ public void testDefaultSSLSettingsCheck() { "Default TLS/SSL settings have been removed", "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + "#tls-setting-fallback", - "Nodes with default TLS/SSL settings: [node_check]"); + "each component must have TLS/SSL configured explicitly"); assertSettingsAndIssue("xpack.ssl.keystore.path", randomAlphaOfLength(8), expected); assertSettingsAndIssue("xpack.ssl.truststore.password", randomAlphaOfLengthBetween(2, 12), expected); assertSettingsAndIssue("xpack.ssl.certificate_authorities",