Merge branch 'trunk' into HDFS-17438

Author: KeeProMise

LICENSE-binary

@@ -241,7 +241,7 @@ com.google.guava:guava:20.0
 com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
-com.nimbusds:nimbus-jose-jwt:9.31
+com.nimbusds:nimbus-jose-jwt:9.37.2
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.4
 commons-cli:commons-cli:1.5.0
@@ -340,7 +340,7 @@ org.apache.sshd:sshd-core:2.11.0
 org.apache.sshd:sshd-sftp:2.11.0
 org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
-org.apache.zookeeper:zookeeper:3.8.3
+org.apache.zookeeper:zookeeper:3.8.4
 org.codehaus.jettison:jettison:1.5.4
 org.eclipse.jetty:jetty-annotations:9.4.53.v20231009
 org.eclipse.jetty:jetty-http:9.4.53.v20231009

dev-support/bin/hadoop.sh

@@ -387,6 +387,11 @@ function personality_modules
       fi
     ;;
     unit)
+      if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 && (-z "$IS_NIGHTLY_BUILD" || "$IS_NIGHTLY_BUILD" == 0) ]]; then
+        echo "Won't run unit tests for Windows in pre-commit CI"
+        return
+      fi
+
       extra="-Dsurefire.rerunFailingTestsCount=2"
       if [[ "${BUILDMODE}" = full ]]; then
         ordering=mvnsrc

hadoop-common-project/hadoop-common/pom.xml

@@ -646,6 +646,7 @@
             <exclude>src/main/native/*</exclude>
             <exclude>src/main/native/config/*</exclude>
             <exclude>src/main/native/m4/*</exclude>
+            <exclude>src/main/winutils/winutils.sln</exclude>
             <exclude>src/test/empty-file</exclude>
             <exclude>src/test/all-tests</exclude>
             <exclude>src/main/native/gtest/**/*</exclude>
@@ -655,6 +656,7 @@
             <exclude>src/test/resources/test.har/_masterindex</exclude>
             <exclude>src/test/resources/test.har/part-0</exclude>
             <exclude>src/test/resources/javakeystoreprovider.password</exclude>
+            <exclude>src/test/resources/lz4/sequencefile</exclude>
             <exclude>dev-support/jdiff-workaround.patch</exclude>
           </excludes>
         </configuration>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java

@@ -23,13 +23,17 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.StringReader;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
 
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
 import org.apache.hadoop.classification.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static java.nio.file.Files.createLink;
 
@@ -50,6 +54,11 @@ public class HardLink {
   private static HardLinkCommandGetter getHardLinkCommand;
 
   public final LinkStats linkStats; //not static
+
+  static final Logger LOG = LoggerFactory.getLogger(HardLink.class);
+
+  private static final String FILE_ATTRIBUTE_VIEW = "unix";
+  private static final String FILE_ATTRIBUTE = "unix:nlink";
 
   //initialize the command "getters" statically, so can use their 
   //methods without instantiating the HardLink object
@@ -204,6 +213,21 @@ public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
     }
   }
 
+    /**
+     * Determines whether the system supports hardlinks.
+     * @param f - file to examine
+     * @return true if hardlinks are supported, false otherwise
+     */
+  public static boolean supportsHardLink(File f) {
+    try {
+      FileStore store = Files.getFileStore(f.toPath());
+      return store.supportsFileAttributeView(FILE_ATTRIBUTE_VIEW);
+    } catch (IOException e) {
+      LOG.warn("Failed to determine if hardlink is supported", e);
+      return false;
+    }
+  }
+
    /**
    * Retrieves the number of links to the specified file.
     *
@@ -220,6 +244,10 @@ public static int getLinkCount(File fileName) throws IOException {
       throw new FileNotFoundException(fileName + " not found.");
     }
 
+    if (supportsHardLink(fileName)) {
+      return (int) Files.getAttribute(fileName.toPath(), FILE_ATTRIBUTE);
+    }
+
     // construct and execute shell command
     String[] cmd = getHardLinkCommand.linkCount(fileName);
     String inpMsg = null;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -1128,8 +1128,7 @@ public void run() {
             synchronized (ipcStreams.out) {
               if (LOG.isDebugEnabled()) {
                 Call call = pair.getLeft();
-                LOG.debug(getName() + "{} sending #{} {}", getName(), call.id,
-                    call.rpcRequest);
+                LOG.debug("{} sending #{} {}", getName(), call.id, call.rpcRequest);
               }
               // RpcRequestHeader + RpcRequest
               ipcStreams.sendRequest(buf.toByteArray());

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -40,6 +40,7 @@
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.log4j.LogManager;
@@ -79,6 +80,18 @@ public class StringUtils {
   public static final Pattern ENV_VAR_PATTERN = Shell.WINDOWS ?
     WIN_ENV_VAR_PATTERN : SHELL_ENV_VAR_PATTERN;
 
+  /**
+   * {@link #getTrimmedStringCollectionSplitByEquals(String)} throws
+   * {@link IllegalArgumentException} with error message starting with this string
+   * if the argument provided is not valid representation of non-empty key-value
+   * pairs.
+   * Value = {@value}
+   */
+  @VisibleForTesting
+  public static final String STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG =
+      "Trimmed string split by equals does not correctly represent "
+          + "non-empty key-value pairs.";
+
   /**
    * Make a string representation of the exception.
    * @param e The exception to stringify
@@ -494,10 +507,19 @@ public static Map<String, String> getTrimmedStringCollectionSplitByEquals(
     String[] trimmedList = getTrimmedStrings(str);
     Map<String, String> pairs = new HashMap<>();
     for (String s : trimmedList) {
-      String[] splitByKeyVal = getTrimmedStringsSplitByEquals(s);
-      if (splitByKeyVal.length == 2) {
-        pairs.put(splitByKeyVal[0], splitByKeyVal[1]);
+      if (s.isEmpty()) {
+        continue;
       }
+      String[] splitByKeyVal = getTrimmedStringsSplitByEquals(s);
+      Preconditions.checkArgument(
+          splitByKeyVal.length == 2,
+          STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG + " Input: " + str);
+      boolean emptyKey = org.apache.commons.lang3.StringUtils.isEmpty(splitByKeyVal[0]);
+      boolean emptyVal = org.apache.commons.lang3.StringUtils.isEmpty(splitByKeyVal[1]);
+      Preconditions.checkArgument(
+          !emptyKey && !emptyVal,
+          STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG + " Input: " + str);
+      pairs.put(splitByKeyVal[0], splitByKeyVal[1]);
     }
     return pairs;
   }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java

@@ -219,6 +219,16 @@ public void testGetLinkCount() throws IOException {
     assertEquals(1, getLinkCount(x3));
   }
 
+  @Test
+  public void testGetLinkCountFromFileAttribute() throws IOException {
+    assertTrue(supportsHardLink(x1));
+    assertEquals(1, getLinkCount(x1));
+    assertTrue(supportsHardLink(x2));
+    assertEquals(1, getLinkCount(x2));
+    assertTrue(supportsHardLink(x3));
+    assertEquals(1, getLinkCount(x3));
+  }
+
   /**
    * Test the single-file method HardLink.createHardLink().
    * Also tests getLinkCount() with values greater than one.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java

@@ -19,6 +19,9 @@
 package org.apache.hadoop.util;
 
 import java.util.Locale;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.util.StringUtils.STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG;
 import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String;
 import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.string2long;
 import static org.junit.Assert.assertArrayEquals;
@@ -515,7 +518,7 @@ public void testCreateStartupShutdownMessage() {
   }
 
   @Test
-  public void testStringCollectionSplitByEquals() {
+  public void testStringCollectionSplitByEqualsSuccess() {
     Map<String, String> splitMap =
         StringUtils.getTrimmedStringCollectionSplitByEquals("");
     Assertions
@@ -566,6 +569,59 @@ public void testStringCollectionSplitByEquals() {
         .containsEntry("element.xyz.key5", "element.abc.val5")
         .containsEntry("element.xyz.key6", "element.abc.val6")
         .containsEntry("element.xyz.key7", "element.abc.val7");
+
+    splitMap = StringUtils.getTrimmedStringCollectionSplitByEquals(
+        "element.first.key1 = element.first.val2 ,element.first.key1 =element.first.val1");
+    Assertions
+        .assertThat(splitMap)
+        .describedAs("Map of key value pairs split by equals(=) and comma(,)")
+        .hasSize(1)
+        .containsEntry("element.first.key1", "element.first.val1");
+
+    splitMap = StringUtils.getTrimmedStringCollectionSplitByEquals(
+        ",,, , ,, ,element.first.key1 = element.first.val2 ,"
+            + "element.first.key1 = element.first.val1 , ,,, ,");
+    Assertions
+        .assertThat(splitMap)
+        .describedAs("Map of key value pairs split by equals(=) and comma(,)")
+        .hasSize(1)
+        .containsEntry("element.first.key1", "element.first.val1");
+
+    splitMap = StringUtils.getTrimmedStringCollectionSplitByEquals(
+        ",, , ,      ,, ,");
+    Assertions
+        .assertThat(splitMap)
+        .describedAs("Map of key value pairs split by equals(=) and comma(,)")
+        .hasSize(0);
+
+  }
+
+  @Test
+  public void testStringCollectionSplitByEqualsFailure() throws Exception {
+    intercept(
+        IllegalArgumentException.class,
+        STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG,
+        () -> StringUtils.getTrimmedStringCollectionSplitByEquals(" = element.abc.val1"));
+
+    intercept(
+        IllegalArgumentException.class,
+        STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG,
+        () -> StringUtils.getTrimmedStringCollectionSplitByEquals("element.abc.key1="));
+
+    intercept(
+        IllegalArgumentException.class,
+        STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG,
+        () -> StringUtils.getTrimmedStringCollectionSplitByEquals("="));
+
+    intercept(
+        IllegalArgumentException.class,
+        STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG,
+        () -> StringUtils.getTrimmedStringCollectionSplitByEquals("== = =    ="));
+
+    intercept(
+        IllegalArgumentException.class,
+        STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG,
+        () -> StringUtils.getTrimmedStringCollectionSplitByEquals(",="));
   }
 
   // Benchmark for StringUtils split

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java

@@ -57,11 +57,16 @@
 import org.apache.hadoop.tracing.TraceUtils;
 
 import org.apache.hadoop.thirdparty.protobuf.Message;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 
 /** Sender */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class Sender implements DataTransferProtocol {
+  private static final Logger LOG = LoggerFactory.getLogger(Sender.class);
+
   private final DataOutputStream out;
 
   /** Create a sender for DataTransferProtocol with a output stream. */

hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml

@@ -172,6 +172,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-params</artifactId>
@@ -187,6 +192,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit-vintage-engine</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>