Skip to content

Commit 2e8b7dc

Browse files
steveloughransteveloughran
committed
Merge branch 'trunk' into s3/HADOOP-19105-vectorio-recovery
Change-Id: I669d2bf55795a091c0d2a3b54a11cd85dc2cee1c
2 parents f00171c + 7e67358 commit 2e8b7dc

File tree

70 files changed

+3129
-658
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+3129
-658
lines changed

‎LICENSE-binary‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -251,43 +251,43 @@ commons-daemon:commons-daemon:1.0.13
251251
commons-io:commons-io:2.16.1
252252
commons-net:commons-net:3.9.0
253253
de.ruedigermoeller:fst:2.50
254-
io.grpc:grpc-api:1.53.0
255-
io.grpc:grpc-context:1.53.0
256-
io.grpc:grpc-core:1.53.0
257-
io.grpc:grpc-netty:1.53.0
258-
io.grpc:grpc-protobuf:1.53.0
259-
io.grpc:grpc-protobuf-lite:1.53.0
260-
io.grpc:grpc-stub:1.53.0
261-
io.netty:netty-all:4.1.100.Final
262-
io.netty:netty-buffer:4.1.100.Final
263-
io.netty:netty-codec:4.1.100.Final
264-
io.netty:netty-codec-dns:4.1.100.Final
265-
io.netty:netty-codec-haproxy:4.1.100.Final
266-
io.netty:netty-codec-http:4.1.100.Final
267-
io.netty:netty-codec-http2:4.1.100.Final
268-
io.netty:netty-codec-memcache:4.1.100.Final
269-
io.netty:netty-codec-mqtt:4.1.100.Final
270-
io.netty:netty-codec-redis:4.1.100.Final
271-
io.netty:netty-codec-smtp:4.1.100.Final
272-
io.netty:netty-codec-socks:4.1.100.Final
273-
io.netty:netty-codec-stomp:4.1.100.Final
274-
io.netty:netty-codec-xml:4.1.100.Final
275-
io.netty:netty-common:4.1.100.Final
276-
io.netty:netty-handler:4.1.100.Final
277-
io.netty:netty-handler-proxy:4.1.100.Final
278-
io.netty:netty-resolver:4.1.100.Final
279-
io.netty:netty-resolver-dns:4.1.100.Final
280-
io.netty:netty-transport:4.1.100.Final
281-
io.netty:netty-transport-rxtx:4.1.100.Final
282-
io.netty:netty-transport-sctp:4.1.100.Final
283-
io.netty:netty-transport-udt:4.1.100.Final
284-
io.netty:netty-transport-classes-epoll:4.1.100.Final
285-
io.netty:netty-transport-native-unix-common:4.1.100.Final
286-
io.netty:netty-transport-classes-kqueue:4.1.100.Final
287-
io.netty:netty-resolver-dns-classes-macos:4.1.100.Final
288-
io.netty:netty-transport-native-epoll:4.1.100.Final
289-
io.netty:netty-transport-native-kqueue:4.1.100.Final
290-
io.netty:netty-resolver-dns-native-macos:4.1.100.Final
254+
io.grpc:grpc-api:1.69.0
255+
io.grpc:grpc-context:1.69.0
256+
io.grpc:grpc-core:1.69.0
257+
io.grpc:grpc-netty:1.69.0
258+
io.grpc:grpc-protobuf:1.69.0
259+
io.grpc:grpc-protobuf-lite:1.69.0
260+
io.grpc:grpc-stub:1.69.0
261+
io.netty:netty-all:4.1.116.Final
262+
io.netty:netty-buffer:4.1.116.Final
263+
io.netty:netty-codec:4.1.116.Final
264+
io.netty:netty-codec-dns:4.1.116.Final
265+
io.netty:netty-codec-haproxy:4.1.116.Final
266+
io.netty:netty-codec-http:4.1.116.Final
267+
io.netty:netty-codec-http2:4.1.116.Final
268+
io.netty:netty-codec-memcache:4.1.116.Final
269+
io.netty:netty-codec-mqtt:4.1.116.Final
270+
io.netty:netty-codec-redis:4.1.116.Final
271+
io.netty:netty-codec-smtp:4.1.116.Final
272+
io.netty:netty-codec-socks:4.1.116.Final
273+
io.netty:netty-codec-stomp:4.1.116.Final
274+
io.netty:netty-codec-xml:4.1.116.Final
275+
io.netty:netty-common:4.1.116.Final
276+
io.netty:netty-handler:4.1.116.Final
277+
io.netty:netty-handler-proxy:4.1.116.Final
278+
io.netty:netty-resolver:4.1.116.Final
279+
io.netty:netty-resolver-dns:4.1.116.Final
280+
io.netty:netty-transport:4.1.116.Final
281+
io.netty:netty-transport-rxtx:4.1.116.Final
282+
io.netty:netty-transport-sctp:4.1.116.Final
283+
io.netty:netty-transport-udt:4.1.116.Final
284+
io.netty:netty-transport-classes-epoll:4.1.116.Final
285+
io.netty:netty-transport-native-unix-common:4.1.116.Final
286+
io.netty:netty-transport-classes-kqueue:4.1.116.Final
287+
io.netty:netty-resolver-dns-classes-macos:4.1.116.Final
288+
io.netty:netty-transport-native-epoll:4.1.116.Final
289+
io.netty:netty-transport-native-kqueue:4.1.116.Final
290+
io.netty:netty-resolver-dns-native-macos:4.1.116.Final
291291
io.opencensus:opencensus-api:0.12.3
292292
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
293293
io.reactivex:rxjava:1.3.8
@@ -486,7 +486,7 @@ org.bouncycastle:bcpkix-jdk18on:1.78.1
486486
org.bouncycastle:bcprov-jdk18on:1.78.1
487487
org.bouncycastle:bcutil-jdk18on:1.78.1
488488
org.checkerframework:checker-qual:3.8.0
489-
org.codehaus.mojo:animal-sniffer-annotations:1.21
489+
org.codehaus.mojo:animal-sniffer-annotations:1.24
490490
org.jruby.jcodings:jcodings:1.0.13
491491
org.jruby.joni:joni:2.1.2
492492
org.ojalgo:ojalgo:43.0

‎dev-support/docker/pkg-resolver/packages.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,11 +261,13 @@
261261
"debian:10": "openjdk-11-jdk",
262262
"ubuntu:focal": [
263263
"openjdk-8-jdk",
264-
"openjdk-11-jdk"
264+
"openjdk-11-jdk",
265+
"openjdk-17-jdk"
265266
],
266267
"ubuntu:focal::arch64": [
267268
"openjdk-8-jdk",
268-
"openjdk-11-jdk"
269+
"openjdk-11-jdk",
270+
"openjdk-17-jdk"
269271
]
270272
},
271273
"pinentry-curses": {

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,11 @@ private int doRun(String[] args)
263263
rpcServer.stopAndJoin();
264264

265265
elector.quitElection(true);
266-
healthMonitor.shutdown();
267-
healthMonitor.join();
266+
267+
if (healthMonitor != null) {
268+
healthMonitor.shutdown();
269+
healthMonitor.join();
270+
}
268271
}
269272
return 0;
270273
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockCompressorStream.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,11 @@ public class BlockCompressorStream extends CompressorStream {
5454
public BlockCompressorStream(OutputStream out, Compressor compressor,
5555
int bufferSize, int compressionOverhead) {
5656
super(out, compressor, bufferSize);
57-
MAX_INPUT_SIZE = bufferSize - compressionOverhead;
57+
if (bufferSize - compressionOverhead >= 0) {
58+
MAX_INPUT_SIZE = bufferSize - compressionOverhead;
59+
} else {
60+
throw new IllegalArgumentException("buffer size is less than compression overhead");
61+
}
5862
}
5963

6064
/**

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.hadoop.classification.InterfaceAudience;
2727
import org.apache.hadoop.classification.InterfaceStability;
2828
import org.apache.hadoop.classification.InterfaceStability.Unstable;
29+
import org.apache.hadoop.classification.VisibleForTesting;
2930
import org.apache.hadoop.conf.Configuration;
3031
import org.apache.hadoop.io.Writable;
3132
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -34,7 +35,6 @@
3435
import org.apache.hadoop.security.UserGroupInformation;
3536
import org.apache.hadoop.security.token.SecretManager;
3637
import org.apache.hadoop.security.token.TokenIdentifier;
37-
import org.apache.hadoop.classification.VisibleForTesting;
3838
import org.apache.hadoop.tracing.TraceScope;
3939
import org.apache.hadoop.tracing.Tracer;
4040
import org.apache.hadoop.util.Time;
@@ -393,28 +393,38 @@ static class ProtobufRpcEngineCallbackImpl
393393
private final RPC.Server server;
394394
private final Call call;
395395
private final String methodName;
396-
private final long setupTime;
397396

398397
public ProtobufRpcEngineCallbackImpl() {
399398
this.server = CURRENT_CALL_INFO.get().getServer();
400399
this.call = Server.getCurCall().get();
401400
this.methodName = CURRENT_CALL_INFO.get().getMethodName();
402-
this.setupTime = Time.now();
401+
}
402+
403+
private void updateProcessingDetails(Call rpcCall, long deltaNanos) {
404+
ProcessingDetails details = rpcCall.getProcessingDetails();
405+
rpcCall.getProcessingDetails().set(ProcessingDetails.Timing.PROCESSING, deltaNanos,
406+
TimeUnit.NANOSECONDS);
407+
deltaNanos -= details.get(ProcessingDetails.Timing.LOCKWAIT, TimeUnit.NANOSECONDS);
408+
deltaNanos -= details.get(ProcessingDetails.Timing.LOCKSHARED, TimeUnit.NANOSECONDS);
409+
deltaNanos -= details.get(ProcessingDetails.Timing.LOCKEXCLUSIVE, TimeUnit.NANOSECONDS);
410+
details.set(ProcessingDetails.Timing.LOCKFREE, deltaNanos, TimeUnit.NANOSECONDS);
403411
}
404412

405413
@Override
406414
public void setResponse(Message message) {
407-
long processingTime = Time.now() - setupTime;
415+
long deltaNanos = Time.monotonicNowNanos() - call.getStartHandleTimestampNanos();
416+
updateProcessingDetails(call, deltaNanos);
408417
call.setDeferredResponse(RpcWritable.wrap(message));
409-
server.updateDeferredMetrics(methodName, processingTime);
418+
server.updateDeferredMetrics(call, methodName, deltaNanos);
410419
}
411420

412421
@Override
413422
public void error(Throwable t) {
414-
long processingTime = Time.now() - setupTime;
415-
String detailedMetricsName = t.getClass().getSimpleName();
416-
server.updateDeferredMetrics(detailedMetricsName, processingTime);
423+
long deltaNanos = Time.monotonicNowNanos() - call.getStartHandleTimestampNanos();
424+
updateProcessingDetails(call, deltaNanos);
417425
call.setDeferredError(t);
426+
String detailedMetricsName = t.getClass().getSimpleName();
427+
server.updateDeferredMetrics(call, detailedMetricsName, deltaNanos);
418428
}
419429
}
420430

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.hadoop.io.Writable;
2626
import org.apache.hadoop.io.retry.RetryPolicy;
2727
import org.apache.hadoop.ipc.Client.ConnectionId;
28+
import org.apache.hadoop.ipc.ProcessingDetails.Timing;
2829
import org.apache.hadoop.ipc.RPC.RpcInvoker;
2930
import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngine2Protos.RequestHeaderProto;
3031
import org.apache.hadoop.security.UserGroupInformation;
@@ -425,28 +426,37 @@ static class ProtobufRpcEngineCallbackImpl
425426
private final RPC.Server server;
426427
private final Call call;
427428
private final String methodName;
428-
private final long setupTime;
429429

430430
ProtobufRpcEngineCallbackImpl() {
431431
this.server = CURRENT_CALL_INFO.get().getServer();
432432
this.call = Server.getCurCall().get();
433433
this.methodName = CURRENT_CALL_INFO.get().getMethodName();
434-
this.setupTime = Time.now();
434+
}
435+
436+
private void updateProcessingDetails(Call rpcCall, long deltaNanos) {
437+
ProcessingDetails details = rpcCall.getProcessingDetails();
438+
rpcCall.getProcessingDetails().set(Timing.PROCESSING, deltaNanos, TimeUnit.NANOSECONDS);
439+
deltaNanos -= details.get(Timing.LOCKWAIT, TimeUnit.NANOSECONDS);
440+
deltaNanos -= details.get(Timing.LOCKSHARED, TimeUnit.NANOSECONDS);
441+
deltaNanos -= details.get(Timing.LOCKEXCLUSIVE, TimeUnit.NANOSECONDS);
442+
details.set(Timing.LOCKFREE, deltaNanos, TimeUnit.NANOSECONDS);
435443
}
436444

437445
@Override
438446
public void setResponse(Message message) {
439-
long processingTime = Time.now() - setupTime;
447+
long deltaNanos = Time.monotonicNowNanos() - call.getStartHandleTimestampNanos();
448+
updateProcessingDetails(call, deltaNanos);
440449
call.setDeferredResponse(RpcWritable.wrap(message));
441-
server.updateDeferredMetrics(methodName, processingTime);
450+
server.updateDeferredMetrics(call, methodName, deltaNanos);
442451
}
443452

444453
@Override
445454
public void error(Throwable t) {
446-
long processingTime = Time.now() - setupTime;
447-
String detailedMetricsName = t.getClass().getSimpleName();
448-
server.updateDeferredMetrics(detailedMetricsName, processingTime);
455+
long deltaNanos = Time.monotonicNowNanos() - call.getStartHandleTimestampNanos();
456+
updateProcessingDetails(call, deltaNanos);
449457
call.setDeferredError(t);
458+
String detailedMetricsName = t.getClass().getSimpleName();
459+
server.updateDeferredMetrics(call, detailedMetricsName, deltaNanos);
450460
}
451461
}
452462

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -351,13 +351,13 @@ public static Server get() {
351351
* after the call returns.
352352
*/
353353
private static final ThreadLocal<Call> CurCall = new ThreadLocal<Call>();
354-
354+
355355
/** @return Get the current call. */
356356
@VisibleForTesting
357357
public static ThreadLocal<Call> getCurCall() {
358358
return CurCall;
359359
}
360-
360+
361361
/**
362362
* Returns the currently active RPC call's sequential ID number. A negative
363363
* call ID indicates an invalid value, such as if there is no currently active
@@ -638,7 +638,8 @@ void updateMetrics(Call call, long processingStartTimeNanos, boolean connDropped
638638
rpcMetrics.addRpcQueueTime(queueTime);
639639

640640
if (call.isResponseDeferred() || connDropped) {
641-
// call was skipped; don't include it in processing metrics
641+
// The call was skipped; don't include it in processing metrics.
642+
// Will update metrics in method updateDeferredMetrics.
642643
return;
643644
}
644645

@@ -668,9 +669,41 @@ void updateMetrics(Call call, long processingStartTimeNanos, boolean connDropped
668669
}
669670
}
670671

671-
void updateDeferredMetrics(String name, long processingTime) {
672+
/**
673+
* Update rpc metrics for defered calls.
674+
* @param call The Rpc Call
675+
* @param name Rpc method name
676+
* @param processingTime processing call in ms unit.
677+
*/
678+
void updateDeferredMetrics(Call call, String name, long processingTime) {
679+
long completionTimeNanos = Time.monotonicNowNanos();
680+
long arrivalTimeNanos = call.timestampNanos;
681+
682+
ProcessingDetails details = call.getProcessingDetails();
683+
long waitTime =
684+
details.get(Timing.LOCKWAIT, rpcMetrics.getMetricsTimeUnit());
685+
long responseTime =
686+
details.get(Timing.RESPONSE, rpcMetrics.getMetricsTimeUnit());
687+
rpcMetrics.addRpcLockWaitTime(waitTime);
688+
rpcMetrics.addRpcProcessingTime(processingTime);
689+
rpcMetrics.addRpcResponseTime(responseTime);
672690
rpcMetrics.addDeferredRpcProcessingTime(processingTime);
673691
rpcDetailedMetrics.addDeferredProcessingTime(name, processingTime);
692+
// don't include lock wait for detailed metrics.
693+
processingTime -= waitTime;
694+
rpcDetailedMetrics.addProcessingTime(name, processingTime);
695+
696+
// Overall processing time is from arrival to completion.
697+
long overallProcessingTime = rpcMetrics.getMetricsTimeUnit()
698+
.convert(completionTimeNanos - arrivalTimeNanos, TimeUnit.NANOSECONDS);
699+
rpcDetailedMetrics.addOverallProcessingTime(name, overallProcessingTime);
700+
callQueue.addResponseTime(name, call, details);
701+
if (isLogSlowRPC()) {
702+
logSlowRpcCalls(name, call, details);
703+
}
704+
if (details.getReturnStatus() == RpcStatusProto.SUCCESS) {
705+
rpcMetrics.incrRpcCallSuccesses();
706+
}
674707
}
675708

676709
/**
@@ -963,6 +996,7 @@ public static class Call implements Schedulable,
963996
final int callId; // the client's call id
964997
final int retryCount; // the retry count of the call
965998
private final long timestampNanos; // time the call was received
999+
protected long startHandleTimestampNanos; // time the call was run
9661000
long responseTimestampNanos; // time the call was served
9671001
private AtomicInteger responseWaitCount = new AtomicInteger(1);
9681002
final RPC.RpcKind rpcKind;
@@ -1167,6 +1201,15 @@ public void setDeferredError(Throwable t) {
11671201
public long getTimestampNanos() {
11681202
return timestampNanos;
11691203
}
1204+
1205+
1206+
public long getStartHandleTimestampNanos() {
1207+
return startHandleTimestampNanos;
1208+
}
1209+
1210+
public void setStartHandleTimestampNanos(long startHandleTimestampNanos) {
1211+
this.startHandleTimestampNanos = startHandleTimestampNanos;
1212+
}
11701213
}
11711214

11721215
/** A RPC extended call queued for handling. */
@@ -1243,6 +1286,7 @@ public Void run() throws Exception {
12431286
}
12441287

12451288
long startNanos = Time.monotonicNowNanos();
1289+
this.setStartHandleTimestampNanos(startNanos);
12461290
Writable value = null;
12471291
ResponseParams responseParams = new ResponseParams();
12481292

@@ -1331,6 +1375,7 @@ void doResponse(Throwable t, RpcStatusProto status) throws IOException {
13311375
* Send a deferred response, ignoring errors.
13321376
*/
13331377
private void sendDeferedResponse() {
1378+
long startNanos = Time.monotonicNowNanos();
13341379
try {
13351380
connection.sendResponse(this);
13361381
} catch (Exception e) {
@@ -1342,6 +1387,8 @@ private void sendDeferedResponse() {
13421387
.currentThread().getName() + ", CallId="
13431388
+ callId + ", hostname=" + getHostAddress());
13441389
}
1390+
getProcessingDetails().set(Timing.RESPONSE,
1391+
Time.monotonicNowNanos() - startNanos, TimeUnit.NANOSECONDS);
13451392
}
13461393

13471394
@Override

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,13 @@ private void getGcUsage(MetricsRecordBuilder rb) {
181181
long count = 0;
182182
long timeMillis = 0;
183183
for (GarbageCollectorMXBean gcBean : gcBeans) {
184+
if (gcBean.getName() != null) {
185+
String name = gcBean.getName();
186+
// JDK-8265136 Skip concurrent phase
187+
if (name.startsWith("ZGC") && name.endsWith("Cycles")) {
188+
continue;
189+
}
190+
}
184191
long c = gcBean.getCollectionCount();
185192
long t = gcBean.getCollectionTime();
186193
MetricsInfo[] gcInfo = getGcInfo(gcBean.getName());

0 commit comments

Comments
 (0)