Skip to content

Commit

Permalink
Run Node deprecation checks locally (#38065)
Browse files Browse the repository at this point in the history
At times, we need to check for usage of deprecated settings in settings
which should not be returned by the NodeInfo API.  This commit changes
the deprecation info API to run all node checks locally so that these
settings can be checked without exposing them via any externally
accessible API.
  • Loading branch information
gwbrown authored Feb 1, 2019
1 parent 13cc7f1 commit 0ee1272
Show file tree
Hide file tree
Showing 14 changed files with 600 additions and 227 deletions.
2 changes: 1 addition & 1 deletion client/rest-high-level/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ integTestCluster {
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
// Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
setting 'xpack.ssl.certificate_authorities', 'testnode.crt'
setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
setting 'indices.lifecycle.poll_interval', '1000ms'
keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
import org.elasticsearch.action.support.master.MasterNodeReadRequest;
Expand All @@ -35,7 +34,6 @@
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -63,6 +61,23 @@ public static <T> List<DeprecationIssue> filterChecks(List<T> checks, Function<T
return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList());
}

private static List<DeprecationIssue> mergeNodeIssues(NodesDeprecationCheckResponse response) {
Map<DeprecationIssue, List<String>> issueListMap = new HashMap<>();
for (NodesDeprecationCheckAction.NodeResponse resp : response.getNodes()) {
for (DeprecationIssue issue : resp.getDeprecationIssues()) {
issueListMap.computeIfAbsent(issue, (key) -> new ArrayList<>()).add(resp.getNode().getName());
}
}

return issueListMap.entrySet().stream()
.map(entry -> {
DeprecationIssue issue = entry.getKey();
String details = issue.getDetails() != null ? issue.getDetails() + " " : "";
return new DeprecationIssue(issue.getLevel(), issue.getMessage(), issue.getUrl(),
details + "(nodes impacted: " + entry.getValue() + ")");
}).collect(Collectors.toList());
}

@Override
public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new RequestBuilder(client, this);
Expand Down Expand Up @@ -167,32 +182,29 @@ public int hashCode() {
* this function will run through all the checks and build out the final list of issues that exist in the
* cluster.
*
* @param nodesInfo The list of {@link NodeInfo} metadata objects for retrieving node-level information
* @param nodesStats The list of {@link NodeStats} metadata objects for retrieving node-level information
* @param state The cluster state
* @param indexNameExpressionResolver Used to resolve indices into their concrete names
* @param indices The list of index expressions to evaluate using `indexNameExpressionResolver`
* @param indicesOptions The options to use when resolving and filtering which indices to check
* @param datafeeds The ml datafeed configurations
* @param clusterSettingsChecks The list of cluster-level checks
* @param nodeSettingsChecks The list of node-level checks
* @param nodeDeprecationResponse The response containing the deprecation issues found on each node
* @param indexSettingsChecks The list of index-level checks that will be run across all specified
* concrete indices
* @param clusterSettingsChecks The list of cluster-level checks
* @param mlSettingsCheck The list of ml checks
* @return The list of deprecation issues found in the cluster
*/
public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List<NodeStats> nodesStats, ClusterState state,
IndexNameExpressionResolver indexNameExpressionResolver,
String[] indices, IndicesOptions indicesOptions,
List<DatafeedConfig> datafeeds,
List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
public static DeprecationInfoAction.Response from(ClusterState state,
IndexNameExpressionResolver indexNameExpressionResolver,
String[] indices, IndicesOptions indicesOptions,
List<DatafeedConfig> datafeeds,
NodesDeprecationCheckResponse nodeDeprecationResponse,
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks,
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
List<DeprecationIssue> clusterSettingsIssues = filterChecks(clusterSettingsChecks,
(c) -> c.apply(state));
List<DeprecationIssue> nodeSettingsIssues = filterChecks(nodeSettingsChecks,
(c) -> c.apply(nodesInfo, nodesStats));
List<DeprecationIssue> nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse);
List<DeprecationIssue> mlSettingsIssues = new ArrayList<>();
for (DatafeedConfig config : datafeeds) {
mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config)));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.core.deprecation;

import org.elasticsearch.action.Action;
import org.elasticsearch.action.support.nodes.BaseNodeRequest;
import org.elasticsearch.action.support.nodes.BaseNodeResponse;
import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;

import java.io.IOException;
import java.util.List;
import java.util.Objects;

/**
* Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings
* can be accessed in the deprecation checks.
*/
public class NodesDeprecationCheckAction extends Action<NodesDeprecationCheckRequest, NodesDeprecationCheckResponse,
NodesDeprecationCheckAction.RequestBuilder> {
public static final NodesDeprecationCheckAction INSTANCE = new NodesDeprecationCheckAction();
public static final String NAME = "cluster:admin/xpack/deprecation/nodes/info";

private NodesDeprecationCheckAction() {
super(NAME);
}

@Override
public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new RequestBuilder(client, INSTANCE, new NodesDeprecationCheckRequest());
}

@Override
public NodesDeprecationCheckResponse newResponse() {
return new NodesDeprecationCheckResponse();
}

public static class NodeRequest extends BaseNodeRequest {

NodesDeprecationCheckRequest request;

public NodeRequest() {}
public NodeRequest(String nodeId, NodesDeprecationCheckRequest request) {
super(nodeId);
this.request = request;
}

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
request = new NodesDeprecationCheckRequest();
request.readFrom(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
request.writeTo(out);
}
}

public static class NodeResponse extends BaseNodeResponse {
private List<DeprecationIssue> deprecationIssues;

public NodeResponse() {
super();
}

public NodeResponse(DiscoveryNode node, List<DeprecationIssue> deprecationIssues) {
super(node);
this.deprecationIssues = deprecationIssues;
}

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
deprecationIssues = in.readList(DeprecationIssue::new);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeList(this.deprecationIssues);
}

public static NodeResponse readNodeResponse(StreamInput in) throws IOException {
NodeResponse nodeResponse = new NodeResponse();
nodeResponse.readFrom(in);
return nodeResponse;
}

public List<DeprecationIssue> getDeprecationIssues() {
return deprecationIssues;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NodeResponse that = (NodeResponse) o;
return Objects.equals(getDeprecationIssues(), that.getDeprecationIssues())
&& Objects.equals(getNode(), that.getNode());
}

@Override
public int hashCode() {
return Objects.hash(getNode(), getDeprecationIssues());
}
}

public static class RequestBuilder extends NodesOperationRequestBuilder<NodesDeprecationCheckRequest,
NodesDeprecationCheckResponse, RequestBuilder> {

protected RequestBuilder(ElasticsearchClient client,
Action<NodesDeprecationCheckRequest, NodesDeprecationCheckResponse, RequestBuilder> action,
NodesDeprecationCheckRequest request) {
super(client, action, request);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.core.deprecation;

import org.elasticsearch.action.support.nodes.BaseNodesRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;

import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;

public class NodesDeprecationCheckRequest extends BaseNodesRequest<NodesDeprecationCheckRequest> {
public NodesDeprecationCheckRequest() {}

public NodesDeprecationCheckRequest(String... nodesIds) {
super(nodesIds);
}

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
}

@Override
public int hashCode() {
return Objects.hash((Object[]) this.nodesIds());
}

@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
NodesDeprecationCheckRequest that = (NodesDeprecationCheckRequest) obj;
return Arrays.equals(this.nodesIds(), that.nodesIds());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.core.deprecation;

import org.elasticsearch.action.FailedNodeException;
import org.elasticsearch.action.support.nodes.BaseNodesResponse;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;

import java.io.IOException;
import java.util.List;
import java.util.Objects;

public class NodesDeprecationCheckResponse extends BaseNodesResponse<NodesDeprecationCheckAction.NodeResponse> {

public NodesDeprecationCheckResponse() {}

public NodesDeprecationCheckResponse(ClusterName clusterName,
List<NodesDeprecationCheckAction.NodeResponse> nodes,
List<FailedNodeException> failures) {
super(clusterName, nodes, failures);
}

@Override
protected List<NodesDeprecationCheckAction.NodeResponse> readNodesFrom(StreamInput in) throws IOException {
return in.readList(NodesDeprecationCheckAction.NodeResponse::readNodeResponse);
}

@Override
protected void writeNodesTo(StreamOutput out, List<NodesDeprecationCheckAction.NodeResponse> nodes) throws IOException {
out.writeStreamableList(nodes);
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NodesDeprecationCheckResponse that = (NodesDeprecationCheckResponse) o;
return Objects.equals(getClusterName(), that.getClusterName())
&& Objects.equals(getNodes(), that.getNodes())
&& Objects.equals(failures(), that.failures());
}

@Override
public int hashCode() {
return Objects.hash(getClusterName(), getNodes(), failures());
}
}
Loading

0 comments on commit 0ee1272

Please sign in to comment.