Skip to content

Commit 1977eea

Browse files
PrateekSanePrateekSane
authored
HDFS-17641. Change badly distributed blocks metric protobuf requirement to optional (#7274)
Note: This is a follow on pr from #7123 that walks back an incompatible change that was made.
1 parent 8662431 commit 1977eea

File tree

5 files changed

+64
-38
lines changed

5 files changed

+64
-38
lines changed

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,22 +38,21 @@ public final class ECBlockGroupStats {
3838
private final long missingBlockGroups;
3939
private final long bytesInFutureBlockGroups;
4040
private final long pendingDeletionBlocks;
41-
private final long badlyDistributedBlocks;
41+
private final Long badlyDistributedBlocks;
4242
private final Long highestPriorityLowRedundancyBlocks;
4343

4444
public ECBlockGroupStats(long lowRedundancyBlockGroups,
4545
long corruptBlockGroups, long missingBlockGroups,
46-
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
47-
long badlyDistributedBlocks) {
46+
long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
4847
this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
4948
bytesInFutureBlockGroups, pendingDeletionBlocks,
50-
badlyDistributedBlocks, null);
49+
null, null);
5150
}
5251

5352
public ECBlockGroupStats(long lowRedundancyBlockGroups,
5453
long corruptBlockGroups, long missingBlockGroups,
5554
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
56-
long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
55+
Long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
5756
this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
5857
this.corruptBlockGroups = corruptBlockGroups;
5958
this.missingBlockGroups = missingBlockGroups;
@@ -84,7 +83,11 @@ public long getPendingDeletionBlocks() {
8483
return pendingDeletionBlocks;
8584
}
8685

87-
public long getBadlyDistributedBlocks() {
86+
public boolean hasBadlyDistributedBlocks() {
87+
return getBadlyDistributedBlocks() != null;
88+
}
89+
90+
public Long getBadlyDistributedBlocks() {
8891
return badlyDistributedBlocks;
8992
}
9093

@@ -107,8 +110,11 @@ public String toString() {
107110
.append(", BytesInFutureBlockGroups=").append(
108111
getBytesInFutureBlockGroups())
109112
.append(", PendingDeletionBlocks=").append(
110-
getPendingDeletionBlocks())
111-
.append(" , BadlyDistributedBlocks=").append(getBadlyDistributedBlocks());
113+
getPendingDeletionBlocks());
114+
if(hasBadlyDistributedBlocks()) {
115+
statsBuilder.append(", BadlyDistributedBlocks=")
116+
.append(getBadlyDistributedBlocks());
117+
}
112118
if (hasHighestPriorityLowRedundancyBlocks()) {
113119
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
114120
.append(getHighestPriorityLowRedundancyBlocks());
@@ -163,6 +169,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
163169
long bytesInFutureBlockGroups = 0;
164170
long pendingDeletionBlocks = 0;
165171
long badlyDistributedBlocks = 0;
172+
boolean hasBadlyDistributedBlocks = false;
166173
long highestPriorityLowRedundancyBlocks = 0;
167174
boolean hasHighestPriorityLowRedundancyBlocks = false;
168175

@@ -172,20 +179,22 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
172179
missingBlockGroups += stat.getMissingBlockGroups();
173180
bytesInFutureBlockGroups += stat.getBytesInFutureBlockGroups();
174181
pendingDeletionBlocks += stat.getPendingDeletionBlocks();
175-
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
182+
if (stat.hasBadlyDistributedBlocks()) {
183+
hasBadlyDistributedBlocks = true;
184+
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
185+
}
176186
if (stat.hasHighestPriorityLowRedundancyBlocks()) {
177187
hasHighestPriorityLowRedundancyBlocks = true;
178188
highestPriorityLowRedundancyBlocks +=
179189
stat.getHighestPriorityLowRedundancyBlocks();
180190
}
181191
}
182-
if (hasHighestPriorityLowRedundancyBlocks) {
192+
if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
183193
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
184194
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
185195
badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
186196
}
187197
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
188-
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
189-
badlyDistributedBlocks);
198+
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
190199
}
191200
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -37,22 +37,22 @@ public final class ReplicatedBlockStats {
3737
private final long missingReplicationOneBlocks;
3838
private final long bytesInFutureBlocks;
3939
private final long pendingDeletionBlocks;
40-
private final long badlyDistributedBlocks;
40+
private final Long badlyDistributedBlocks;
4141
private final Long highestPriorityLowRedundancyBlocks;
4242

4343
public ReplicatedBlockStats(long lowRedundancyBlocks,
4444
long corruptBlocks, long missingBlocks,
4545
long missingReplicationOneBlocks, long bytesInFutureBlocks,
46-
long pendingDeletionBlocks, long badlyDistributedBlocks) {
46+
long pendingDeletionBlocks) {
4747
this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
4848
missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
49-
badlyDistributedBlocks, null);
49+
null, null);
5050
}
5151

5252
public ReplicatedBlockStats(long lowRedundancyBlocks,
5353
long corruptBlocks, long missingBlocks,
5454
long missingReplicationOneBlocks, long bytesInFutureBlocks,
55-
long pendingDeletionBlocks, long badlyDistributedBlocks,
55+
long pendingDeletionBlocks, Long badlyDistributedBlocks,
5656
Long highestPriorityLowRedundancyBlocks) {
5757
this.lowRedundancyBlocks = lowRedundancyBlocks;
5858
this.corruptBlocks = corruptBlocks;
@@ -89,7 +89,11 @@ public long getPendingDeletionBlocks() {
8989
return pendingDeletionBlocks;
9090
}
9191

92-
public long getBadlyDistributedBlocks() {
92+
public boolean hasBadlyDistributedBlocks() {
93+
return getBadlyDistributedBlocks() != null;
94+
}
95+
96+
public Long getBadlyDistributedBlocks() {
9397
return badlyDistributedBlocks;
9498
}
9599

@@ -113,8 +117,10 @@ public String toString() {
113117
getMissingReplicationOneBlocks())
114118
.append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
115119
.append(", PendingDeletionBlocks=").append(
116-
getPendingDeletionBlocks())
117-
.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
120+
getPendingDeletionBlocks());
121+
if (hasBadlyDistributedBlocks()) {
122+
statsBuilder.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
123+
}
118124
if (hasHighestPriorityLowRedundancyBlocks()) {
119125
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
120126
getHighestPriorityLowRedundancyBlocks());
@@ -137,6 +143,7 @@ public static ReplicatedBlockStats merge(
137143
long bytesInFutureBlocks = 0;
138144
long pendingDeletionBlocks = 0;
139145
long badlyDistributedBlocks = 0;
146+
boolean hasBadlyDistributedBlocks = false;
140147
long highestPriorityLowRedundancyBlocks = 0;
141148
boolean hasHighestPriorityLowRedundancyBlocks = false;
142149

@@ -148,20 +155,24 @@ public static ReplicatedBlockStats merge(
148155
missingReplicationOneBlocks += stat.getMissingReplicationOneBlocks();
149156
bytesInFutureBlocks += stat.getBytesInFutureBlocks();
150157
pendingDeletionBlocks += stat.getPendingDeletionBlocks();
151-
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
158+
159+
if (stat.hasBadlyDistributedBlocks()) {
160+
hasBadlyDistributedBlocks = true;
161+
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
162+
}
152163
if (stat.hasHighestPriorityLowRedundancyBlocks()) {
153164
hasHighestPriorityLowRedundancyBlocks = true;
154165
highestPriorityLowRedundancyBlocks +=
155166
stat.getHighestPriorityLowRedundancyBlocks();
156167
}
157168
}
158-
if (hasHighestPriorityLowRedundancyBlocks) {
169+
if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
159170
return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
160171
missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
161172
pendingDeletionBlocks, badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
162173
}
163174
return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
164175
missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
165-
pendingDeletionBlocks, badlyDistributedBlocks);
176+
pendingDeletionBlocks);
166177
}
167178
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2033,7 +2033,7 @@ public static long[] convert(GetFsStatsResponseProto res) {
20332033

20342034
public static ReplicatedBlockStats convert(
20352035
GetFsReplicatedBlockStatsResponseProto res) {
2036-
if (res.hasHighestPrioLowRedundancyBlocks()) {
2036+
if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
20372037
return new ReplicatedBlockStats(res.getLowRedundancy(),
20382038
res.getCorruptBlocks(), res.getMissingBlocks(),
20392039
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
@@ -2043,21 +2043,20 @@ public static ReplicatedBlockStats convert(
20432043
return new ReplicatedBlockStats(res.getLowRedundancy(),
20442044
res.getCorruptBlocks(), res.getMissingBlocks(),
20452045
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
2046-
res.getBadlyDistributedBlocks(), res.getPendingDeletionBlocks());
2046+
res.getPendingDeletionBlocks());
20472047
}
20482048

20492049
public static ECBlockGroupStats convert(
20502050
GetFsECBlockGroupStatsResponseProto res) {
2051-
if (res.hasHighestPrioLowRedundancyBlocks()) {
2051+
if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
20522052
return new ECBlockGroupStats(res.getLowRedundancy(),
20532053
res.getCorruptBlocks(), res.getMissingBlocks(),
20542054
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
20552055
res.getBadlyDistributedBlocks(), res.getHighestPrioLowRedundancyBlocks());
20562056
}
20572057
return new ECBlockGroupStats(res.getLowRedundancy(),
20582058
res.getCorruptBlocks(), res.getMissingBlocks(),
2059-
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
2060-
res.getBadlyDistributedBlocks());
2059+
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
20612060
}
20622061

20632062
public static DatanodeReportTypeProto convert(DatanodeReportType t) {
@@ -2526,8 +2525,10 @@ public static GetFsReplicatedBlockStatsResponseProto convert(
25262525
replicatedBlockStats.getBytesInFutureBlocks());
25272526
result.setPendingDeletionBlocks(
25282527
replicatedBlockStats.getPendingDeletionBlocks());
2529-
result.setBadlyDistributedBlocks(
2530-
replicatedBlockStats.getBadlyDistributedBlocks());
2528+
if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
2529+
result.setBadlyDistributedBlocks(
2530+
replicatedBlockStats.getBadlyDistributedBlocks());
2531+
}
25312532
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
25322533
result.setHighestPrioLowRedundancyBlocks(
25332534
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
@@ -2547,8 +2548,10 @@ public static GetFsECBlockGroupStatsResponseProto convert(
25472548
ecBlockGroupStats.getBytesInFutureBlockGroups());
25482549
result.setPendingDeletionBlocks(
25492550
ecBlockGroupStats.getPendingDeletionBlocks());
2550-
result.setBadlyDistributedBlocks(
2551-
ecBlockGroupStats.getBadlyDistributedBlocks());
2551+
if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
2552+
result.setBadlyDistributedBlocks(
2553+
ecBlockGroupStats.getBadlyDistributedBlocks());
2554+
}
25522555
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
25532556
result.setHighestPrioLowRedundancyBlocks(
25542557
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -372,8 +372,7 @@ message GetFsReplicatedBlockStatsResponseProto {
372372
required uint64 blocks_in_future = 5;
373373
required uint64 pending_deletion_blocks = 6;
374374
optional uint64 highest_prio_low_redundancy_blocks = 7;
375-
required uint64 badly_distributed_blocks = 8;
376-
375+
optional uint64 badly_distributed_blocks = 8;
377376
}
378377

379378
message GetFsECBlockGroupStatsRequestProto { // no input paramters
@@ -386,7 +385,7 @@ message GetFsECBlockGroupStatsResponseProto {
386385
required uint64 blocks_in_future = 4;
387386
required uint64 pending_deletion_blocks = 5;
388387
optional uint64 highest_prio_low_redundancy_blocks = 6;
389-
required uint64 badly_distributed_blocks = 7;
388+
optional uint64 badly_distributed_blocks = 7;
390389
}
391390

392391
enum DatanodeReportTypeProto { // type of the datanode report

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -554,8 +554,10 @@ public void report(String[] argv, int i) throws IOException {
554554
replicatedBlockStats.getMissingReplicaBlocks());
555555
System.out.println("\tMissing blocks (with replication factor 1): " +
556556
replicatedBlockStats.getMissingReplicationOneBlocks());
557-
System.out.println("\tBadly Distributed Blocks: " +
558-
replicatedBlockStats.getBadlyDistributedBlocks());
557+
if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
558+
System.out.println("\tBadly Distributed Blocks: " +
559+
replicatedBlockStats.getBadlyDistributedBlocks());
560+
}
559561
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
560562
System.out.println("\tLow redundancy blocks with highest priority " +
561563
"to recover: " +
@@ -573,8 +575,10 @@ public void report(String[] argv, int i) throws IOException {
573575
ecBlockGroupStats.getCorruptBlockGroups());
574576
System.out.println("\tMissing block groups: " +
575577
ecBlockGroupStats.getMissingBlockGroups());
576-
System.out.println("\tBadly Distributed Blocks: " +
577-
ecBlockGroupStats.getBadlyDistributedBlocks());
578+
if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
579+
System.out.println("\tBadly Distributed Blocks: " +
580+
ecBlockGroupStats.getBadlyDistributedBlocks());
581+
}
578582
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
579583
System.out.println("\tLow redundancy blocks with highest priority " +
580584
"to recover: " +

0 commit comments

Comments
 (0)