Merge branch 'trunk' into build/HADOOP-19696-hadoop-cloud-connector-dist

Author: steveloughran

dev-support/jenkins.sh

@@ -124,9 +124,8 @@ function run_ci() {
     YETUS_ARGS+=("--mvn-custom-repos")
     YETUS_ARGS+=("--dockermemlimit=22g")
 
-    # test with Java 8 and 11
-    YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
-    YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
+    # test with Java 17
+    YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-17-openjdk-amd64")
     YETUS_ARGS+=("--multijdktests=compile")
   fi
 

hadoop-common-project/hadoop-common/src/CMakeLists.txt

@@ -157,6 +157,8 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x
   set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c")
 elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
   set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_aarch64.c")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^riscv64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^riscv32")
+  set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_riscv.c")
 else()
   message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
 endif()

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -23,6 +23,7 @@
 import java.io.InterruptedIOException;
 import java.io.PrintStream;
 import java.net.BindException;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
 import java.net.URI;
@@ -549,25 +550,50 @@ public HttpServer2 build() throws IOException {
       }
 
       for (URI ep : endpoints) {
-        final ServerConnector connector;
+        //
+        // To enable dual-stack or IPv6 support, use InetAddress
+        // .getAllByName(hostname) to resolve the IP addresses of a host.
+        // When the system property java.net.preferIPv4Stack is set to true,
+        // only IPv4 addresses are returned, and any IPv6 addresses are
+        // ignored, so no extra check is needed to exclude IPv6.
+        // When java.net.preferIPv4Stack is false, both IPv4 and IPv6
+        // addresses may be returned, and any IPv6 addresses will also be
+        // added as connectors.
+        // To disable IPv4, you need to configure the OS at the system level.
+        //
+        InetAddress[] addresses = InetAddress.getAllByName(ep.getHost());
+        server = addConnectors(
+            ep, addresses, server, httpConfig, backlogSize, idleTimeout);
+      }
+      server.loadListeners();
+      return server;
+    }
+
+    @VisibleForTesting
+    HttpServer2 addConnectors(
+        URI ep, InetAddress[] addresses, HttpServer2 server,
+        HttpConfiguration httpConfig, int backlogSize, int idleTimeout){
+      for (InetAddress addr : addresses) {
+        ServerConnector connector;
         String scheme = ep.getScheme();
         if (HTTP_SCHEME.equals(scheme)) {
-          connector = createHttpChannelConnector(server.webServer,
-              httpConfig);
+          connector = createHttpChannelConnector(
+              server.webServer, httpConfig);
         } else if (HTTPS_SCHEME.equals(scheme)) {
-          connector = createHttpsChannelConnector(server.webServer,
-              httpConfig);
+          connector = createHttpsChannelConnector(
+              server.webServer, httpConfig);
         } else {
           throw new HadoopIllegalArgumentException(
               "unknown scheme for endpoint:" + ep);
         }
-        connector.setHost(ep.getHost());
+        LOG.debug("Adding connector to WebServer for address {}",
+            addr.getHostAddress());
+        connector.setHost(addr.getHostAddress());
         connector.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
         connector.setAcceptQueueSize(backlogSize);
         connector.setIdleTimeout(idleTimeout);
         server.addListener(connector);
       }
-      server.loadListeners();
       return server;
     }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -2673,7 +2673,7 @@ private RpcSaslProto buildSaslNegotiateResponse()
       // accelerate token negotiation by sending initial challenge
       // in the negotiation response
       if (enabledAuthMethods.contains(AuthMethod.TOKEN)
-          && SaslMechanismFactory.isDefaultMechanism(AuthMethod.TOKEN.getMechanismName())) {
+          && SaslMechanismFactory.isDigestMechanism(AuthMethod.TOKEN.getMechanismName())) {
         saslServer = createSaslServer(AuthMethod.TOKEN);
         byte[] challenge = saslServer.evaluateResponse(new byte[0]);
         RpcSaslProto.Builder negotiateBuilder =

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CustomizedCallbackHandler.java

@@ -47,7 +47,7 @@ private static synchronized CustomizedCallbackHandler getSynchronously(
 
       //cache miss
       final Class<?> clazz = conf.getClass(key, DefaultHandler.class);
-      LOG.info("{} = {}", key, clazz);
+      LOG.debug("{} = {}", key, clazz);
       if (clazz == DefaultHandler.class) {
         return DefaultHandler.INSTANCE;
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslMechanismFactory.java

@@ -65,5 +65,13 @@ public static boolean isDefaultMechanism(String saslMechanism) {
     return HADOOP_SECURITY_SASL_MECHANISM_DEFAULT.equals(saslMechanism);
   }
 
+  public static boolean isDigestMechanism(String saslMechanism) {
+    return saslMechanism.startsWith("DIGEST-");
+  }
+
   private SaslMechanismFactory() {}
+
+  public static void main(String[] args) {
+    System.out.println("SASL_MECHANISM = " + getMechanism());
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java

@@ -26,6 +26,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
+import java.util.function.ToIntFunction;
 
 /**
  * Encapsulates logic for composing multiple CRCs into one or more combined CRCs
@@ -35,11 +36,11 @@
  */
 @InterfaceAudience.LimitedPrivate({"Common", "HDFS", "MapReduce", "Yarn"})
 @InterfaceStability.Unstable
-public class CrcComposer {
+public final class CrcComposer {
   private static final int CRC_SIZE_BYTES = 4;
   private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class);
 
-  private final int crcPolynomial;
+  private final ToIntFunction<Long> mod;
   private final int precomputedMonomialForHint;
   private final long bytesPerCrcHint;
   private final long stripeLength;
@@ -79,28 +80,14 @@ public static CrcComposer newCrcComposer(
    */
   public static CrcComposer newStripedCrcComposer(
       DataChecksum.Type type, long bytesPerCrcHint, long stripeLength) {
-    int polynomial = DataChecksum.getCrcPolynomialForType(type);
-    return new CrcComposer(
-        polynomial,
-        CrcUtil.getMonomial(bytesPerCrcHint, polynomial),
-        bytesPerCrcHint,
-        stripeLength);
+    return new CrcComposer(type, bytesPerCrcHint, stripeLength);
   }
 
-  CrcComposer(
-      int crcPolynomial,
-      int precomputedMonomialForHint,
-      long bytesPerCrcHint,
-      long stripeLength) {
-    LOG.debug(
-        "crcPolynomial=0x{}, precomputedMonomialForHint=0x{}, "
-        + "bytesPerCrcHint={}, stripeLength={}",
-        Integer.toString(crcPolynomial, 16),
-        Integer.toString(precomputedMonomialForHint, 16),
-        bytesPerCrcHint,
-        stripeLength);
-    this.crcPolynomial = crcPolynomial;
-    this.precomputedMonomialForHint = precomputedMonomialForHint;
+  private CrcComposer(DataChecksum.Type type, long bytesPerCrcHint, long stripeLength) {
+    LOG.debug("type={}, bytesPerCrcHint={}, stripeLength={}",
+        type, bytesPerCrcHint, stripeLength);
+    this.mod = DataChecksum.getModFunction(type);
+    this.precomputedMonomialForHint = CrcUtil.getMonomial(bytesPerCrcHint, mod);
     this.bytesPerCrcHint = bytesPerCrcHint;
     this.stripeLength = stripeLength;
   }
@@ -161,10 +148,10 @@ public void update(int crcB, long bytesPerCrc) {
       curCompositeCrc = crcB;
     } else if (bytesPerCrc == bytesPerCrcHint) {
       curCompositeCrc = CrcUtil.composeWithMonomial(
-          curCompositeCrc, crcB, precomputedMonomialForHint, crcPolynomial);
+          curCompositeCrc, crcB, precomputedMonomialForHint, mod);
     } else {
       curCompositeCrc = CrcUtil.compose(
-          curCompositeCrc, crcB, bytesPerCrc, crcPolynomial);
+          curCompositeCrc, crcB, bytesPerCrc, mod);
     }
 
     curPositionInStripe += bytesPerCrc;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 
 import java.util.Arrays;
+import java.util.function.ToIntFunction;
 
 /**
  * This class provides utilities for working with CRCs.
@@ -32,6 +33,55 @@ public final class CrcUtil {
   public static final int MULTIPLICATIVE_IDENTITY = 0x80000000;
   public static final int GZIP_POLYNOMIAL = 0xEDB88320;
   public static final int CASTAGNOLI_POLYNOMIAL = 0x82F63B78;
+  private static final long UNIT = 0x8000_0000_0000_0000L;
+
+  /**
+   * @return a * b (mod p),
+   *         where mod p is computed by the given mod function.
+   */
+  static int multiplyMod(int a, int b, ToIntFunction<Long> mod) {
+    final long left  = ((long)a) << 32;
+    final long right = ((long)b) << 32;
+
+    final long product
+        = ((((((left & (UNIT /*  */)) == 0L? 0L : right)
+        ^     ((left & (UNIT >>>  1)) == 0L? 0L : right >>>  1))
+        ^    (((left & (UNIT >>>  2)) == 0L? 0L : right >>>  2)
+        ^     ((left & (UNIT >>>  3)) == 0L? 0L : right >>>  3)))
+        ^   ((((left & (UNIT >>>  4)) == 0L? 0L : right >>>  4)
+        ^     ((left & (UNIT >>>  5)) == 0L? 0L : right >>>  5))
+        ^    (((left & (UNIT >>>  6)) == 0L? 0L : right >>>  6)
+        ^     ((left & (UNIT >>>  7)) == 0L? 0L : right >>>  7))))
+
+        ^  (((((left & (UNIT >>>  8)) == 0L? 0L : right >>>  8)
+        ^     ((left & (UNIT >>>  9)) == 0L? 0L : right >>>  9))
+        ^    (((left & (UNIT >>> 10)) == 0L? 0L : right >>> 10)
+        ^     ((left & (UNIT >>> 11)) == 0L? 0L : right >>> 11)))
+        ^   ((((left & (UNIT >>> 12)) == 0L? 0L : right >>> 12)
+        ^     ((left & (UNIT >>> 13)) == 0L? 0L : right >>> 13))
+        ^    (((left & (UNIT >>> 14)) == 0L? 0L : right >>> 14)
+        ^     ((left & (UNIT >>> 15)) == 0L? 0L : right >>> 15)))))
+
+        ^ ((((((left & (UNIT >>> 16)) == 0L? 0L : right >>> 16)
+        ^     ((left & (UNIT >>> 17)) == 0L? 0L : right >>> 17))
+        ^    (((left & (UNIT >>> 18)) == 0L? 0L : right >>> 18)
+        ^     ((left & (UNIT >>> 19)) == 0L? 0L : right >>> 19)))
+        ^   ((((left & (UNIT >>> 20)) == 0L? 0L : right >>> 20)
+        ^     ((left & (UNIT >>> 21)) == 0L? 0L : right >>> 21))
+        ^    (((left & (UNIT >>> 22)) == 0L? 0L : right >>> 22)
+        ^     ((left & (UNIT >>> 23)) == 0L? 0L : right >>> 23))))
+
+        ^  (((((left & (UNIT >>> 24)) == 0L? 0L : right >>> 24)
+        ^     ((left & (UNIT >>> 25)) == 0L? 0L : right >>> 25))
+        ^    (((left & (UNIT >>> 26)) == 0L? 0L : right >>> 26)
+        ^     ((left & (UNIT >>> 27)) == 0L? 0L : right >>> 27)))
+        ^   ((((left & (UNIT >>> 28)) == 0L? 0L : right >>> 28)
+        ^     ((left & (UNIT >>> 29)) == 0L? 0L : right >>> 29))
+        ^    (((left & (UNIT >>> 30)) == 0L? 0L : right >>> 30)
+        ^     ((left & (UNIT >>> 31)) == 0L? 0L : right >>> 31)))));
+
+    return mod.applyAsInt(product);
+  }
 
   /**
    * Hide default constructor for a static utils class.
@@ -48,7 +98,7 @@ private CrcUtil() {
    * @param mod mod.
    * @return monomial.
    */
-  public static int getMonomial(long lengthBytes, int mod) {
+  public static int getMonomial(long lengthBytes, ToIntFunction<Long> mod) {
     if (lengthBytes == 0) {
       return MULTIPLICATIVE_IDENTITY;
     } else if (lengthBytes < 0) {
@@ -67,9 +117,9 @@ public static int getMonomial(long lengthBytes, int mod) {
     while (degree > 0) {
       if ((degree & 1) != 0) {
         product = (product == MULTIPLICATIVE_IDENTITY) ? multiplier :
-            galoisFieldMultiply(product, multiplier, mod);
+            multiplyMod(product, multiplier, mod);
       }
-      multiplier = galoisFieldMultiply(multiplier, multiplier, mod);
+      multiplier = multiplyMod(multiplier, multiplier, mod);
       degree >>= 1;
     }
     return product;
@@ -85,8 +135,8 @@ public static int getMonomial(long lengthBytes, int mod) {
    * @return compose with monomial.
    */
   public static int composeWithMonomial(
-      int crcA, int crcB, int monomial, int mod) {
-    return galoisFieldMultiply(crcA, monomial, mod) ^ crcB;
+      int crcA, int crcB, int monomial, ToIntFunction<Long> mod) {
+    return multiplyMod(crcA, monomial, mod) ^ crcB;
   }
 
   /**
@@ -98,7 +148,7 @@ public static int composeWithMonomial(
    * @param mod mod.
    * @return compose result.
    */
-  public static int compose(int crcA, int crcB, long lengthB, int mod) {
+  public static int compose(int crcA, int crcB, long lengthB, ToIntFunction<Long> mod) {
     int monomial = getMonomial(lengthB, mod);
     return composeWithMonomial(crcA, crcB, monomial, mod);
   }
@@ -199,40 +249,5 @@ public static String toMultiCrcString(final byte[] bytes) {
     return sb.toString();
   }
 
-  /**
-   * Galois field multiplication of {@code p} and {@code q} with the
-   * generator polynomial {@code m} as the modulus.
-   *
-   * @param m The little-endian polynomial to use as the modulus when
-   *     multiplying p and q, with implicit "1" bit beyond the bottom bit.
-   */
-  private static int galoisFieldMultiply(int p, int q, int m) {
-    int summation = 0;
-
-    // Top bit is the x^0 place; each right-shift increments the degree of the
-    // current term.
-    int curTerm = MULTIPLICATIVE_IDENTITY;
-
-    // Iteratively multiply p by x mod m as we go to represent the q[i] term
-    // (of degree x^i) times p.
-    int px = p;
-
-    while (curTerm != 0) {
-      if ((q & curTerm) != 0) {
-        summation ^= px;
-      }
 
-      // Bottom bit represents highest degree since we're little-endian; before
-      // we multiply by "x" for the next term, check bottom bit to know whether
-      // the resulting px will thus have a term matching the implicit "1" term
-      // of "m" and thus will need to subtract "m" after mutiplying by "x".
-      boolean hasMaxDegree = ((px & 1) != 0);
-      px >>>= 1;
-      if (hasMaxDegree) {
-        px ^= m;
-      }
-      curTerm >>>= 1;
-    }
-    return summation;
-  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -22,6 +22,7 @@
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.function.ToIntFunction;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
@@ -118,15 +119,14 @@ static Checksum newCrc32C() {
    * @return the int representation of the polynomial associated with the
    *     CRC {@code type}, suitable for use with further CRC arithmetic.
    */
-  public static int getCrcPolynomialForType(Type type) {
+  static ToIntFunction<Long> getModFunction(Type type) {
     switch (type) {
     case CRC32:
-      return CrcUtil.GZIP_POLYNOMIAL;
+      return PureJavaCrc32::mod;
     case CRC32C:
-      return CrcUtil.CASTAGNOLI_POLYNOMIAL;
+      return PureJavaCrc32C::mod;
     default:
-      throw new IllegalArgumentException(
-          "No CRC polynomial could be associated with type: " + type);
+      throw new IllegalArgumentException("Unexpected type: " + type);
     }
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -90,6 +90,18 @@ public void update(final byte[] b, final int offset, final int len) {
     crc = localCrc;
   }
 
+  /**
+   * Compute x mod p, where p is the CRC32 polynomial.
+   * @param x the input value
+   * @return x mod p
+   */
+  public static int mod(long x) {
+    final int y = (int)(x);
+    return (int)(x >> 32)
+           ^ ((T[((y << 24) >>> 24) + 0x300] ^ T[((y << 16) >>> 24) + 0x200])
+           ^  (T[((y <<  8) >>> 24) + 0x100] ^ T[((y /* */) >>> 24) /*   */]));
+  }
+
   @Override
   final public void update(int b) {
     crc = (crc >>> 8) ^ T[(((crc ^ b) << 24) >>> 24)];