Skip to content

Commit 93fbc42

Browse files
steveloughransteveloughran
committed
Merge branch 'trunk' into s3/HADOOP-19712-EvaluatingStatisticsMap-deadlock
2 parents c3c15b8 + 3424214 commit 93fbc42

File tree

43 files changed

+1115
-179
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1115
-179
lines changed

‎dev-support/docker/pkg-resolver/packages.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,20 +264,20 @@
264264
"openjdk-17-jdk"
265265
],
266266
"ubuntu:focal": [
267-
"temurin-24-jdk",
267+
"temurin-25-jdk",
268268
"openjdk-8-jdk",
269269
"openjdk-11-jdk",
270270
"openjdk-17-jdk",
271271
"openjdk-21-jdk"
272272
],
273273
"ubuntu:noble": [
274-
"temurin-24-jdk",
274+
"temurin-25-jdk",
275275
"openjdk-11-jdk",
276276
"openjdk-17-jdk",
277277
"openjdk-21-jdk"
278278
],
279279
"ubuntu:focal::arch64": [
280-
"temurin-24-jdk",
280+
"temurin-25-jdk",
281281
"openjdk-8-jdk",
282282
"openjdk-11-jdk",
283283
"openjdk-17-jdk",

‎dev-support/jenkins.sh‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,8 @@ function run_ci() {
124124
YETUS_ARGS+=("--mvn-custom-repos")
125125
YETUS_ARGS+=("--dockermemlimit=22g")
126126

127-
# test with Java 8 and 11
128-
YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
129-
YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
127+
# test with Java 17
128+
YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-17-openjdk-amd64")
130129
YETUS_ARGS+=("--multijdktests=compile")
131130
fi
132131

‎hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Iterables.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
package org.apache.hadoop.fs.tosfs.util;
2020

2121
import org.apache.hadoop.util.Preconditions;
22-
import org.checkerframework.checker.nullness.qual.Nullable;
22+
import org.apache.hadoop.thirdparty.org.checkerframework.checker.nullness.qual.Nullable;
2323

2424
import java.util.Iterator;
2525
import java.util.NoSuchElementException;

‎hadoop-common-project/hadoop-common/src/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,8 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x
157157
set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c")
158158
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
159159
set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_aarch64.c")
160+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^riscv64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^riscv32")
161+
set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_riscv.c")
160162
else()
161163
message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
162164
endif()

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.io.InterruptedIOException;
2424
import java.io.PrintStream;
2525
import java.net.BindException;
26+
import java.net.InetAddress;
2627
import java.net.InetSocketAddress;
2728
import java.net.MalformedURLException;
2829
import java.net.URI;
@@ -549,25 +550,50 @@ public HttpServer2 build() throws IOException {
549550
}
550551

551552
for (URI ep : endpoints) {
552-
final ServerConnector connector;
553+
//
554+
// To enable dual-stack or IPv6 support, use InetAddress
555+
// .getAllByName(hostname) to resolve the IP addresses of a host.
556+
// When the system property java.net.preferIPv4Stack is set to true,
557+
// only IPv4 addresses are returned, and any IPv6 addresses are
558+
// ignored, so no extra check is needed to exclude IPv6.
559+
// When java.net.preferIPv4Stack is false, both IPv4 and IPv6
560+
// addresses may be returned, and any IPv6 addresses will also be
561+
// added as connectors.
562+
// To disable IPv4, you need to configure the OS at the system level.
563+
//
564+
InetAddress[] addresses = InetAddress.getAllByName(ep.getHost());
565+
server = addConnectors(
566+
ep, addresses, server, httpConfig, backlogSize, idleTimeout);
567+
}
568+
server.loadListeners();
569+
return server;
570+
}
571+
572+
@VisibleForTesting
573+
HttpServer2 addConnectors(
574+
URI ep, InetAddress[] addresses, HttpServer2 server,
575+
HttpConfiguration httpConfig, int backlogSize, int idleTimeout){
576+
for (InetAddress addr : addresses) {
577+
ServerConnector connector;
553578
String scheme = ep.getScheme();
554579
if (HTTP_SCHEME.equals(scheme)) {
555-
connector = createHttpChannelConnector(server.webServer,
556-
httpConfig);
580+
connector = createHttpChannelConnector(
581+
server.webServer, httpConfig);
557582
} else if (HTTPS_SCHEME.equals(scheme)) {
558-
connector = createHttpsChannelConnector(server.webServer,
559-
httpConfig);
583+
connector = createHttpsChannelConnector(
584+
server.webServer, httpConfig);
560585
} else {
561586
throw new HadoopIllegalArgumentException(
562587
"unknown scheme for endpoint:" + ep);
563588
}
564-
connector.setHost(ep.getHost());
589+
LOG.debug("Adding connector to WebServer for address {}",
590+
addr.getHostAddress());
591+
connector.setHost(addr.getHostAddress());
565592
connector.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
566593
connector.setAcceptQueueSize(backlogSize);
567594
connector.setIdleTimeout(idleTimeout);
568595
server.addListener(connector);
569596
}
570-
server.loadListeners();
571597
return server;
572598
}
573599

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2673,7 +2673,7 @@ private RpcSaslProto buildSaslNegotiateResponse()
26732673
// accelerate token negotiation by sending initial challenge
26742674
// in the negotiation response
26752675
if (enabledAuthMethods.contains(AuthMethod.TOKEN)
2676-
&& SaslMechanismFactory.isDefaultMechanism(AuthMethod.TOKEN.getMechanismName())) {
2676+
&& SaslMechanismFactory.isDigestMechanism(AuthMethod.TOKEN.getMechanismName())) {
26772677
saslServer = createSaslServer(AuthMethod.TOKEN);
26782678
byte[] challenge = saslServer.evaluateResponse(new byte[0]);
26792679
RpcSaslProto.Builder negotiateBuilder =

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CustomizedCallbackHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ private static synchronized CustomizedCallbackHandler getSynchronously(
4747

4848
//cache miss
4949
final Class<?> clazz = conf.getClass(key, DefaultHandler.class);
50-
LOG.info("{} = {}", key, clazz);
50+
LOG.debug("{} = {}", key, clazz);
5151
if (clazz == DefaultHandler.class) {
5252
return DefaultHandler.INSTANCE;
5353
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslMechanismFactory.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,5 +65,13 @@ public static boolean isDefaultMechanism(String saslMechanism) {
6565
return HADOOP_SECURITY_SASL_MECHANISM_DEFAULT.equals(saslMechanism);
6666
}
6767

68+
public static boolean isDigestMechanism(String saslMechanism) {
69+
return saslMechanism.startsWith("DIGEST-");
70+
}
71+
6872
private SaslMechanismFactory() {}
73+
74+
public static void main(String[] args) {
75+
System.out.println("SASL_MECHANISM = " + getMechanism());
76+
}
6977
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.io.ByteArrayOutputStream;
2727
import java.io.DataInputStream;
2828
import java.io.IOException;
29+
import java.util.function.ToIntFunction;
2930

3031
/**
3132
* Encapsulates logic for composing multiple CRCs into one or more combined CRCs
@@ -35,11 +36,11 @@
3536
*/
3637
@InterfaceAudience.LimitedPrivate({"Common", "HDFS", "MapReduce", "Yarn"})
3738
@InterfaceStability.Unstable
38-
public class CrcComposer {
39+
public final class CrcComposer {
3940
private static final int CRC_SIZE_BYTES = 4;
4041
private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class);
4142

42-
private final int crcPolynomial;
43+
private final ToIntFunction<Long> mod;
4344
private final int precomputedMonomialForHint;
4445
private final long bytesPerCrcHint;
4546
private final long stripeLength;
@@ -79,28 +80,14 @@ public static CrcComposer newCrcComposer(
7980
*/
8081
public static CrcComposer newStripedCrcComposer(
8182
DataChecksum.Type type, long bytesPerCrcHint, long stripeLength) {
82-
int polynomial = DataChecksum.getCrcPolynomialForType(type);
83-
return new CrcComposer(
84-
polynomial,
85-
CrcUtil.getMonomial(bytesPerCrcHint, polynomial),
86-
bytesPerCrcHint,
87-
stripeLength);
83+
return new CrcComposer(type, bytesPerCrcHint, stripeLength);
8884
}
8985

90-
CrcComposer(
91-
int crcPolynomial,
92-
int precomputedMonomialForHint,
93-
long bytesPerCrcHint,
94-
long stripeLength) {
95-
LOG.debug(
96-
"crcPolynomial=0x{}, precomputedMonomialForHint=0x{}, "
97-
+ "bytesPerCrcHint={}, stripeLength={}",
98-
Integer.toString(crcPolynomial, 16),
99-
Integer.toString(precomputedMonomialForHint, 16),
100-
bytesPerCrcHint,
101-
stripeLength);
102-
this.crcPolynomial = crcPolynomial;
103-
this.precomputedMonomialForHint = precomputedMonomialForHint;
86+
private CrcComposer(DataChecksum.Type type, long bytesPerCrcHint, long stripeLength) {
87+
LOG.debug("type={}, bytesPerCrcHint={}, stripeLength={}",
88+
type, bytesPerCrcHint, stripeLength);
89+
this.mod = DataChecksum.getModFunction(type);
90+
this.precomputedMonomialForHint = CrcUtil.getMonomial(bytesPerCrcHint, mod);
10491
this.bytesPerCrcHint = bytesPerCrcHint;
10592
this.stripeLength = stripeLength;
10693
}
@@ -161,10 +148,10 @@ public void update(int crcB, long bytesPerCrc) {
161148
curCompositeCrc = crcB;
162149
} else if (bytesPerCrc == bytesPerCrcHint) {
163150
curCompositeCrc = CrcUtil.composeWithMonomial(
164-
curCompositeCrc, crcB, precomputedMonomialForHint, crcPolynomial);
151+
curCompositeCrc, crcB, precomputedMonomialForHint, mod);
165152
} else {
166153
curCompositeCrc = CrcUtil.compose(
167-
curCompositeCrc, crcB, bytesPerCrc, crcPolynomial);
154+
curCompositeCrc, crcB, bytesPerCrc, mod);
168155
}
169156

170157
curPositionInStripe += bytesPerCrc;

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java‎

Lines changed: 56 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.hadoop.classification.InterfaceStability;
2323

2424
import java.util.Arrays;
25+
import java.util.function.ToIntFunction;
2526

2627
/**
2728
* This class provides utilities for working with CRCs.
@@ -32,6 +33,55 @@ public final class CrcUtil {
3233
public static final int MULTIPLICATIVE_IDENTITY = 0x80000000;
3334
public static final int GZIP_POLYNOMIAL = 0xEDB88320;
3435
public static final int CASTAGNOLI_POLYNOMIAL = 0x82F63B78;
36+
private static final long UNIT = 0x8000_0000_0000_0000L;
37+
38+
/**
39+
* @return a * b (mod p),
40+
* where mod p is computed by the given mod function.
41+
*/
42+
static int multiplyMod(int a, int b, ToIntFunction<Long> mod) {
43+
final long left = ((long)a) << 32;
44+
final long right = ((long)b) << 32;
45+
46+
final long product
47+
= ((((((left & (UNIT /* */)) == 0L? 0L : right)
48+
^ ((left & (UNIT >>> 1)) == 0L? 0L : right >>> 1))
49+
^ (((left & (UNIT >>> 2)) == 0L? 0L : right >>> 2)
50+
^ ((left & (UNIT >>> 3)) == 0L? 0L : right >>> 3)))
51+
^ ((((left & (UNIT >>> 4)) == 0L? 0L : right >>> 4)
52+
^ ((left & (UNIT >>> 5)) == 0L? 0L : right >>> 5))
53+
^ (((left & (UNIT >>> 6)) == 0L? 0L : right >>> 6)
54+
^ ((left & (UNIT >>> 7)) == 0L? 0L : right >>> 7))))
55+
56+
^ (((((left & (UNIT >>> 8)) == 0L? 0L : right >>> 8)
57+
^ ((left & (UNIT >>> 9)) == 0L? 0L : right >>> 9))
58+
^ (((left & (UNIT >>> 10)) == 0L? 0L : right >>> 10)
59+
^ ((left & (UNIT >>> 11)) == 0L? 0L : right >>> 11)))
60+
^ ((((left & (UNIT >>> 12)) == 0L? 0L : right >>> 12)
61+
^ ((left & (UNIT >>> 13)) == 0L? 0L : right >>> 13))
62+
^ (((left & (UNIT >>> 14)) == 0L? 0L : right >>> 14)
63+
^ ((left & (UNIT >>> 15)) == 0L? 0L : right >>> 15)))))
64+
65+
^ ((((((left & (UNIT >>> 16)) == 0L? 0L : right >>> 16)
66+
^ ((left & (UNIT >>> 17)) == 0L? 0L : right >>> 17))
67+
^ (((left & (UNIT >>> 18)) == 0L? 0L : right >>> 18)
68+
^ ((left & (UNIT >>> 19)) == 0L? 0L : right >>> 19)))
69+
^ ((((left & (UNIT >>> 20)) == 0L? 0L : right >>> 20)
70+
^ ((left & (UNIT >>> 21)) == 0L? 0L : right >>> 21))
71+
^ (((left & (UNIT >>> 22)) == 0L? 0L : right >>> 22)
72+
^ ((left & (UNIT >>> 23)) == 0L? 0L : right >>> 23))))
73+
74+
^ (((((left & (UNIT >>> 24)) == 0L? 0L : right >>> 24)
75+
^ ((left & (UNIT >>> 25)) == 0L? 0L : right >>> 25))
76+
^ (((left & (UNIT >>> 26)) == 0L? 0L : right >>> 26)
77+
^ ((left & (UNIT >>> 27)) == 0L? 0L : right >>> 27)))
78+
^ ((((left & (UNIT >>> 28)) == 0L? 0L : right >>> 28)
79+
^ ((left & (UNIT >>> 29)) == 0L? 0L : right >>> 29))
80+
^ (((left & (UNIT >>> 30)) == 0L? 0L : right >>> 30)
81+
^ ((left & (UNIT >>> 31)) == 0L? 0L : right >>> 31)))));
82+
83+
return mod.applyAsInt(product);
84+
}
3585

3686
/**
3787
* Hide default constructor for a static utils class.
@@ -48,7 +98,7 @@ private CrcUtil() {
4898
* @param mod mod.
4999
* @return monomial.
50100
*/
51-
public static int getMonomial(long lengthBytes, int mod) {
101+
public static int getMonomial(long lengthBytes, ToIntFunction<Long> mod) {
52102
if (lengthBytes == 0) {
53103
return MULTIPLICATIVE_IDENTITY;
54104
} else if (lengthBytes < 0) {
@@ -67,9 +117,9 @@ public static int getMonomial(long lengthBytes, int mod) {
67117
while (degree > 0) {
68118
if ((degree & 1) != 0) {
69119
product = (product == MULTIPLICATIVE_IDENTITY) ? multiplier :
70-
galoisFieldMultiply(product, multiplier, mod);
120+
multiplyMod(product, multiplier, mod);
71121
}
72-
multiplier = galoisFieldMultiply(multiplier, multiplier, mod);
122+
multiplier = multiplyMod(multiplier, multiplier, mod);
73123
degree >>= 1;
74124
}
75125
return product;
@@ -85,8 +135,8 @@ public static int getMonomial(long lengthBytes, int mod) {
85135
* @return compose with monomial.
86136
*/
87137
public static int composeWithMonomial(
88-
int crcA, int crcB, int monomial, int mod) {
89-
return galoisFieldMultiply(crcA, monomial, mod) ^ crcB;
138+
int crcA, int crcB, int monomial, ToIntFunction<Long> mod) {
139+
return multiplyMod(crcA, monomial, mod) ^ crcB;
90140
}
91141

92142
/**
@@ -98,7 +148,7 @@ public static int composeWithMonomial(
98148
* @param mod mod.
99149
* @return compose result.
100150
*/
101-
public static int compose(int crcA, int crcB, long lengthB, int mod) {
151+
public static int compose(int crcA, int crcB, long lengthB, ToIntFunction<Long> mod) {
102152
int monomial = getMonomial(lengthB, mod);
103153
return composeWithMonomial(crcA, crcB, monomial, mod);
104154
}
@@ -199,40 +249,5 @@ public static String toMultiCrcString(final byte[] bytes) {
199249
return sb.toString();
200250
}
201251

202-
/**
203-
* Galois field multiplication of {@code p} and {@code q} with the
204-
* generator polynomial {@code m} as the modulus.
205-
*
206-
* @param m The little-endian polynomial to use as the modulus when
207-
* multiplying p and q, with implicit "1" bit beyond the bottom bit.
208-
*/
209-
private static int galoisFieldMultiply(int p, int q, int m) {
210-
int summation = 0;
211-
212-
// Top bit is the x^0 place; each right-shift increments the degree of the
213-
// current term.
214-
int curTerm = MULTIPLICATIVE_IDENTITY;
215-
216-
// Iteratively multiply p by x mod m as we go to represent the q[i] term
217-
// (of degree x^i) times p.
218-
int px = p;
219-
220-
while (curTerm != 0) {
221-
if ((q & curTerm) != 0) {
222-
summation ^= px;
223-
}
224252

225-
// Bottom bit represents highest degree since we're little-endian; before
226-
// we multiply by "x" for the next term, check bottom bit to know whether
227-
// the resulting px will thus have a term matching the implicit "1" term
228-
// of "m" and thus will need to subtract "m" after mutiplying by "x".
229-
boolean hasMaxDegree = ((px & 1) != 0);
230-
px >>>= 1;
231-
if (hasMaxDegree) {
232-
px ^= m;
233-
}
234-
curTerm >>>= 1;
235-
}
236-
return summation;
237-
}
238253
}

0 commit comments

Comments
 (0)