Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.x] Improve Node Shutdown Observability (#78727) #78865

Merged
merged 4 commits into from
Oct 13, 2021
Merged

[7.x] Improve Node Shutdown Observability (#78727) #78865

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5d4d843
Improve Node Shutdown Observability (#78727)
gwbrown Oct 8, 2021
cc4c903
Remove references to targetNodeName as that isn't in 7.15
gwbrown Oct 13, 2021
95f3430
Merge remote-tracking branch 'elastic/7.x' into backport/7.x/pr-78727
gwbrown Oct 13, 2021
54ad0cc
Update BWC version
gwbrown Oct 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 35 additions & 3 deletions server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

package org.elasticsearch.cluster.metadata;

import org.elasticsearch.Version;
import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
Expand All @@ -20,25 +22,42 @@
import java.util.Objects;

public class ShutdownShardMigrationStatus implements Writeable, ToXContentObject {
private static final Version ALLOCATION_DECISION_ADDED_VERSION = Version.V_7_16_0;

private final SingleNodeShutdownMetadata.Status status;
private final long shardsRemaining;
@Nullable private final String explanation;
@Nullable private final ShardAllocationDecision allocationDecision;

public ShutdownShardMigrationStatus(SingleNodeShutdownMetadata.Status status, long shardsRemaining) {
this(status, shardsRemaining, null);
this(status, shardsRemaining, null, null);
}

public ShutdownShardMigrationStatus(SingleNodeShutdownMetadata.Status status, long shardsRemaining, @Nullable String explanation) {
this(status, shardsRemaining, explanation, null);
}

public ShutdownShardMigrationStatus(
SingleNodeShutdownMetadata.Status status,
long shardsRemaining,
@Nullable String explanation,
@Nullable ShardAllocationDecision allocationDecision
) {
this.status = Objects.requireNonNull(status, "status must not be null");
this.shardsRemaining = shardsRemaining;
this.explanation = explanation;
this.allocationDecision = allocationDecision;
}

public ShutdownShardMigrationStatus(StreamInput in) throws IOException {
this.status = in.readEnum(SingleNodeShutdownMetadata.Status.class);
this.shardsRemaining = in.readLong();
this.explanation = in.readOptionalString();
if (in.getVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
this.allocationDecision = in.readOptionalWriteable(ShardAllocationDecision::new);
} else {
this.allocationDecision = null;
}
}

public long getShardsRemaining() {
Expand All @@ -61,6 +80,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (Objects.nonNull(explanation)) {
builder.field("explanation", explanation);
}
if (Objects.nonNull(allocationDecision)) {
builder.startObject("node_allocation_decision");
{
allocationDecision.toXContent(builder, params);
}
builder.endObject();
}
builder.endObject();
return builder;
}
Expand All @@ -70,19 +96,25 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(status);
out.writeLong(shardsRemaining);
out.writeOptionalString(explanation);
if (out.getVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
out.writeOptionalWriteable(allocationDecision);
}
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if ((o instanceof ShutdownShardMigrationStatus) == false) return false;
ShutdownShardMigrationStatus that = (ShutdownShardMigrationStatus) o;
return shardsRemaining == that.shardsRemaining && status == that.status && Objects.equals(explanation, that.explanation);
return shardsRemaining == that.shardsRemaining
&& status == that.status
&& Objects.equals(explanation, that.explanation)
&& Objects.equals(allocationDecision, that.allocationDecision);
}

@Override
public int hashCode() {
return Objects.hash(status, shardsRemaining, explanation);
return Objects.hash(status, shardsRemaining, explanation, allocationDecision);
}

@Override
Expand Down
22 changes: 22 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,28 @@ public int hashCode() {
);
}

@Override public String toString() {
final StringBuilder stringBuilder = new StringBuilder();
stringBuilder
.append("{")
.append("nodeId=[")
.append(nodeId)
.append(']')
.append(", type=[")
.append(type)
.append("], reason=[")
.append(reason)
.append(']');
if (allocationDelay != null) {
stringBuilder
.append(", allocationDelay=[")
.append(allocationDelay)
.append("]");
}
stringBuilder.append("}");
return stringBuilder.toString();
}

public static Builder builder() {
return new Builder();
}
Expand Down
2 changes: 2 additions & 0 deletions ...own/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ protected void masterOperation(
public ClusterState execute(ClusterState currentState) throws Exception {
NodesShutdownMetadata currentShutdownMetadata = currentState.metadata().custom(NodesShutdownMetadata.TYPE);

logger.info("removing shutdown record for node [{}]", request.getNodeId());

return ClusterState.builder(currentState)
.metadata(
Metadata.builder(currentState.metadata())
Expand Down
24 changes: 12 additions & 12 deletions ...down/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,17 +260,15 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
return hasShardCopyOnOtherNode == false;
})
.peek(pair -> {
if (logger.isTraceEnabled()) { // don't serialize the decision unless we have to
logger.trace(
"node [{}] shutdown of type [{}] stalled: found shard [{}][{}] from index [{}] with negative decision: [{}]",
nodeId,
shutdownType,
pair.v1().getId(),
pair.v1().primary() ? "primary" : "replica",
pair.v1().shardId().getIndexName(),
Strings.toString(pair.v2())
);
}
logger.debug(
"node [{}] shutdown of type [{}] stalled: found shard [{}][{}] from index [{}] with negative decision: [{}]",
nodeId,
shutdownType,
pair.v1().getId(),
pair.v1().primary() ? "primary" : "replica",
pair.v1().shardId().getIndexName(),
Strings.toString(pair.v2())
);
})
.findFirst();

Expand All @@ -285,6 +283,7 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
} else if (unmovableShard.isPresent()) {
// We found a shard that can't be moved, so shard relocation is stalled. Blame the unmovable shard.
ShardRouting shardRouting = unmovableShard.get().v1();
ShardAllocationDecision decision = unmovableShard.get().v2();

return new ShutdownShardMigrationStatus(
SingleNodeShutdownMetadata.Status.STALLED,
Expand All @@ -294,7 +293,8 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
shardRouting.shardId().getId(),
shardRouting.primary() ? "primary" : "replica",
shardRouting.index().getName()
).getFormattedMessage()
).getFormattedMessage(),
decision
);
} else {
return new ShutdownShardMigrationStatus(SingleNodeShutdownMetadata.Status.IN_PROGRESS, totalRemainingShards);
Expand Down
21 changes: 8 additions & 13 deletions ...utdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,19 +64,6 @@ public ClusterState execute(ClusterState currentState) {
currentShutdownMetadata = new NodesShutdownMetadata(new HashMap<>());
}

// Verify that there's not already a shutdown metadata for this node
SingleNodeShutdownMetadata existingRecord = currentShutdownMetadata.getAllNodeMetadataMap().get(request.getNodeId());
if (existingRecord != null) {
logger.info(
"updating existing shutdown record for node [{}] of type [{}] with reason [{}] with new type [{}] and reason [{}]",
existingRecord.getNodeId(),
existingRecord.getType(),
existingRecord.getReason(),
request.getType(),
request.getReason()
);
}

final boolean nodeSeen = currentState.getNodes().nodeExists(request.getNodeId());

SingleNodeShutdownMetadata newNodeMetadata = SingleNodeShutdownMetadata.builder()
Expand All @@ -89,6 +76,14 @@ public ClusterState execute(ClusterState currentState) {
.setTargetNodeName(request.getTargetNodeName())
.build();

// Verify that there's not already a shutdown metadata for this node
SingleNodeShutdownMetadata existingRecord = currentShutdownMetadata.getAllNodeMetadataMap().get(request.getNodeId());
if (existingRecord != null) {
logger.info("updating existing shutdown record {} with new record {}", existingRecord, newNodeMetadata);
} else {
logger.info("creating shutdown record {}", newNodeMetadata);
}

return ClusterState.builder(currentState)
.metadata(
Metadata.builder(currentState.metadata())
Expand Down