Merge branch 'trunk' into HADOOP-19406_UDSSupport

Author: manika137

BUILDING.txt

@@ -635,11 +635,12 @@ hadoop-dist/target upon successful build. Run these commands from an
 "x64 Native Tools Command Prompt for VS 2019" which can be found under "Visual Studio 2019" in the
 Windows start menu. If you're using the Docker image from Dockerfile_windows_10, you'll be
 logged into "x64 Native Tools Command Prompt for VS 2019" automatically when you start the
-container.
+container. The Docker image does not have a full VS install, so you need to add the
+-Dskip.platformToolsetDetection option (already included below in the examples).
 
 > set classpath=
 > set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
-> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist^
+> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist -Dskip.platformToolsetDetection^
     -Drequire.openssl -Drequire.test.libhadoop -Pyarn-ui -Dshell-executable=C:\Git\bin\bash.exe^
     -Dtar -Dopenssl.prefix=C:\vcpkg\installed\x64-windows^
     -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows^
@@ -651,7 +652,7 @@ Assuming that we're still running in the Docker container hadoop-windows-10-buil
 following command to create the Apache Hadoop release tarball -
 
 > set IS_WINDOWS=1
-> set MVN_ARGS="-Dshell-executable=C:\Git\bin\bash.exe -Dhttps.protocols=TLSv1.2 -Pnative-win -Drequire.openssl -Dopenssl.prefix=C:\vcpkg\installed\x64-windows -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo -Dwindows.build.hdfspp.dll=off -Duse.platformToolsetVersion=v142 -Dwindows.no.sasl=on -DskipTests -DskipDocs -Drequire.test.libhadoop"
+> set MVN_ARGS="-Dshell-executable=C:\Git\bin\bash.exe -Dhttps.protocols=TLSv1.2 -Pnative-win -Dskip.platformToolsetDetection -Drequire.openssl -Dopenssl.prefix=C:\vcpkg\installed\x64-windows -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo -Dwindows.build.hdfspp.dll=off -Duse.platformToolsetVersion=v142 -Dwindows.no.sasl=on -DskipTests -DskipDocs -Drequire.test.libhadoop"
 > C:\Git\bin\bash.exe C:\hadoop\dev-support\bin\create-release --mvnargs=%MVN_ARGS%
 
 Note:

LICENSE-binary

@@ -218,12 +218,12 @@ com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.12.7
-com.fasterxml.jackson.core:jackson-core:2.12.7
-com.fasterxml.jackson.core:jackson-databind:2.12.7.1
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
+com.fasterxml.jackson.core:jackson-annotations:2.14.3
+com.fasterxml.jackson.core:jackson-core:2.14.3
+com.fasterxml.jackson.core:jackson-databind:2.14.3
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.3
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.3
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.3
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.ben-manes.caffeine:caffeine:2.9.3
@@ -298,7 +298,6 @@ javax.inject:javax.inject:1
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 org.apache.avro:avro:1.11.4
-org.apache.avro:avro:1.11.3
 org.apache.commons:commons-compress:1.26.1
 org.apache.commons:commons-configuration2:2.10.1
 org.apache.commons:commons-csv:1.9.0
@@ -505,8 +504,7 @@ javax.cache:cache-api:1.1.1
 javax.servlet:javax.servlet-api:3.1.0
 javax.servlet.jsp:jsp-api:2.1
 javax.websocket:javax.websocket-api:1.0
-javax.ws.rs:jsr311-api:1.1.1
-javax.xml.bind:jaxb-api:2.2.11
+javax.xml.bind:jaxb-api:2.3.1
 
 Eclipse Distribution License (EDL) 1.0
 --------------------------

dev-support/docker/Dockerfile_windows_10

@@ -49,20 +49,13 @@ RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\G
 # hadolint ignore=DL3003
 RUN powershell git clone https://github.com/microsoft/vcpkg.git \
     && cd vcpkg \
-    && git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
+    && git fetch --all \
+    && git checkout 2025.03.19 \
     && .\bootstrap-vcpkg.bat
 
-# Fix wrong download link for 7-zip in .\vcpkg\scripts\vcpkgTools.xml:
-# - https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0 (not working anymore) is replaced with
-# - https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0 (working)
-# Replacing is done using the Linux tool "sed".
-RUN choco install sed -y
-RUN sed -i "s,https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0,https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0,g" .\vcpkg\scripts\vcpkgTools.xml
-
-RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
+ADD vcpkg/vcpkg.json .
+
+RUN powershell .\vcpkg\vcpkg.exe install --x-install-root .\vcpkg\installed
 
 # Install Azul Java 8 JDK.
 RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip

dev-support/docker/vcpkg/vcpkg.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
+  "dependencies": [
+    "boost",
+    "protobuf",
+    "openssl",
+    "zlib"
+  ],
+  "builtin-baseline": "289a69379604112a433874fe8b9812dad3103341",
+  "overrides": [
+    {
+      "name": "protobuf",
+      "version": "3.21.12"
+    }
+  ]
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java

@@ -162,7 +162,7 @@ public boolean moveToTrash(Path path) throws IOException {
           LOG.warn("Can't create(mkdir) trash directory: " + baseTrashPath);
           return false;
         }
-      } catch (FileAlreadyExistsException e) {
+      } catch (FileAlreadyExistsException | ParentNotDirectoryException e) {
         // find the path which is not a directory, and modify baseTrashPath
         // & trashPath, then mkdirs
         Path existsFilePath = baseTrashPath;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java

@@ -415,7 +415,7 @@ public void setResponse(Message message) {
         long deltaNanos = Time.monotonicNowNanos() - call.getStartHandleTimestampNanos();
         updateProcessingDetails(call, deltaNanos);
         call.setDeferredResponse(RpcWritable.wrap(message));
-        server.updateDeferredMetrics(call, methodName, deltaNanos);
+        server.updateDeferredMetrics(call, methodName);
       }
 
       @Override
@@ -424,7 +424,7 @@ public void error(Throwable t) {
         updateProcessingDetails(call, deltaNanos);
         call.setDeferredError(t);
         String detailedMetricsName = t.getClass().getSimpleName();
-        server.updateDeferredMetrics(call, detailedMetricsName, deltaNanos);
+        server.updateDeferredMetrics(call, detailedMetricsName);
       }
     }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java

@@ -447,7 +447,7 @@ public void setResponse(Message message) {
         long deltaNanos = Time.monotonicNowNanos() - call.getStartHandleTimestampNanos();
         updateProcessingDetails(call, deltaNanos);
         call.setDeferredResponse(RpcWritable.wrap(message));
-        server.updateDeferredMetrics(call, methodName, deltaNanos);
+        server.updateDeferredMetrics(call, methodName);
       }
 
       @Override
@@ -456,7 +456,7 @@ public void error(Throwable t) {
         updateProcessingDetails(call, deltaNanos);
         call.setDeferredError(t);
         String detailedMetricsName = t.getClass().getSimpleName();
-        server.updateDeferredMetrics(call, detailedMetricsName, deltaNanos);
+        server.updateDeferredMetrics(call, detailedMetricsName);
       }
     }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -673,9 +673,8 @@ void updateMetrics(Call call, long processingStartTimeNanos, boolean connDropped
    * Update rpc metrics for defered calls.
    * @param call The Rpc Call
    * @param name Rpc method name
-   * @param processingTime processing call in ms unit.
    */
-  void updateDeferredMetrics(Call call, String name, long processingTime) {
+  void updateDeferredMetrics(Call call, String name) {
     long completionTimeNanos = Time.monotonicNowNanos();
     long arrivalTimeNanos = call.timestampNanos;
 
@@ -684,6 +683,8 @@ void updateDeferredMetrics(Call call, String name, long processingTime) {
         details.get(Timing.LOCKWAIT, rpcMetrics.getMetricsTimeUnit());
     long responseTime =
         details.get(Timing.RESPONSE, rpcMetrics.getMetricsTimeUnit());
+    long processingTime =
+        details.get(Timing.PROCESSING, rpcMetrics.getMetricsTimeUnit());
     rpcMetrics.addRpcLockWaitTime(waitTime);
     rpcMetrics.addRpcProcessingTime(processingTime);
     rpcMetrics.addRpcResponseTime(responseTime);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java

@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.lang.reflect.Constructor;
 import java.net.BindException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -46,10 +45,6 @@
 
 import javax.net.SocketFactory;
 
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
-
 import org.apache.commons.net.util.SubnetUtils;
 import org.apache.commons.net.util.SubnetUtils.SubnetInfo;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -58,9 +53,13 @@
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
+import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.dynamic.DynConstructors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -941,10 +940,59 @@ public static IOException wrapException(final String destHost,
 
       }
     } catch (IOException ex) {
-      return (IOException) new IOException("Failed on local exception: "
-          + exception + "; Host Details : "
-          + getHostDetailsAsString(destHost, destPort, localHost))
-          .initCause(exception);
+      try {
+        return new IOException("Failed on local exception: "
+            + exception + "; Host Details : "
+            + getHostDetailsAsString(destHost, destPort, localHost), exception);
+      } catch (Exception ignore) {
+        // in worst case, return the original exception
+        return exception;
+      }
+    }
+  }
+
+  /**
+   * Return an @{@link IOException} of the same type as the input exception but with
+   * a modified exception message that includes the node name.
+   *
+   * @param ioe existing exception.
+   * @param nodeName name of the node.
+   * @return IOException
+   */
+  public static IOException addNodeNameToIOException(final IOException ioe, final String nodeName) {
+    try {
+      final Throwable cause = ioe.getCause();
+      IOException newIoe = null;
+      if (cause != null) {
+        try {
+          DynConstructors.Ctor<? extends IOException> ctor =
+              new DynConstructors.Builder()
+                  .impl(ioe.getClass(), String.class, Throwable.class)
+                  .buildChecked();
+          newIoe = ctor.newInstance(nodeName + ": " + ioe.getMessage(), cause);
+        } catch (NoSuchMethodException e) {
+          // no matching constructor - try next approach below
+        }
+      }
+      if (newIoe == null) {
+        DynConstructors.Ctor<? extends IOException> ctor =
+            new DynConstructors.Builder()
+                .impl(ioe.getClass(), String.class)
+                .buildChecked();
+        newIoe = ctor.newInstance(nodeName + ": " + ioe.getMessage());
+        if (cause != null) {
+          try {
+            newIoe.initCause(cause);
+          } catch (Exception e) {
+            // Unable to initCause. Ignore the exception.
+          }
+        }
+      }
+      newIoe.setStackTrace(ioe.getStackTrace());
+      return newIoe;
+    } catch (Exception e) {
+      // Unable to create new exception. Return the original exception.
+      return ioe;
     }
   }
 
@@ -957,9 +1005,22 @@ private static <T extends IOException> T wrapWithMessage(
       T exception, String msg) throws T {
     Class<? extends Throwable> clazz = exception.getClass();
     try {
-      Constructor<? extends Throwable> ctor = clazz.getConstructor(String.class);
-      Throwable t = ctor.newInstance(msg);
-      return (T)(t.initCause(exception));
+      try {
+        DynConstructors.Ctor<T> ctor =
+            new DynConstructors.Builder()
+                .impl(clazz, String.class, Throwable.class)
+                .buildChecked();
+        return ctor.newInstance(msg, exception);
+      } catch (NoSuchMethodException e) {
+        // no matching constructor - try next approach below
+      }
+      DynConstructors.Ctor<T> ctor =
+          new DynConstructors.Builder()
+              .impl(clazz, String.class)
+              .buildChecked();
+      T newException = ctor.newInstance(msg);
+      newException.initCause(exception);
+      return newException;
     } catch (NoSuchMethodException e) {
       return exception;
     } catch (Throwable e) {
@@ -1114,7 +1175,7 @@ public static Set<Integer> getFreeSocketPorts(int numOfPorts) {
 
   /**
    * Return an @{@link InetAddress} to bind to. If bindWildCardAddress is true
-   * than returns null.
+   * then returns null.
    *
    * @param localAddr local addr.
    * @param bindWildCardAddress bind wildcard address.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java

@@ -24,21 +24,20 @@
 import java.util.StringTokenizer;
 
 import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
 import org.slf4j.event.Level;
 
 import static org.apache.hadoop.fs.FileContextTestHelper.*;
 import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * <p>
@@ -52,7 +51,7 @@
  * test and override {@link #setUp()} to initialize the <code>fc</code> 
  * {@link FileContext} instance variable.
  * 
- * Since this a junit 4 you can also do a single setup before 
+ * Since this a junit 4+ you can also do a single setup before
  * the start of any tests.
  * E.g.
  *     @BeforeClass   public static void clusterSetupAtBegining()
@@ -80,22 +79,22 @@ protected FileContextTestHelper getFileContextHelper() {
 
   protected abstract FileContext getFileContext() throws Exception;
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     fileContextTestHelper = getFileContextHelper();
     fc = getFileContext();
     fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     fc.delete(fileContextTestHelper.getTestRootPath(fc), true);
   }
 
   private void cleanupFile(FileContext fc, Path name) throws IOException {
-    Assert.assertTrue(exists(fc, name));
+    assertTrue(exists(fc, name));
     fc.delete(name, true);
-    Assert.assertTrue(!exists(fc, name));
+    assertTrue(!exists(fc, name));
   }
 
   @Test
@@ -158,12 +157,12 @@ public void testSetOwner() throws IOException {
     try {
       String g0 = groups.get(0);
       fc.setOwner(f, null, g0);
-      Assert.assertEquals(g0, fc.getFileStatus(f).getGroup());
+      assertEquals(fc.getFileStatus(f).getGroup(), g0);
 
       if (groups.size() > 1) {
         String g1 = groups.get(1);
         fc.setOwner(f, null, g1);
-        Assert.assertEquals(g1, fc.getFileStatus(f).getGroup());
+        assertEquals(fc.getFileStatus(f).getGroup(), g1);
       } else {
         System.out.println("Not testing changing the group since user " +
                            "belongs to only one group.");
@@ -193,7 +192,7 @@ public FileContext run() throws Exception {
       }
 
     });
-    assertEquals("otherUser",newFc.getUgi().getUserName());
+    assertEquals(newFc.getUgi().getUserName(), "otherUser");
   }
 
   static List<String> getGroups() throws IOException {
@@ -207,7 +206,7 @@ static List<String> getGroups() throws IOException {
 
 
   void doFilePermissionCheck(FsPermission expectedPerm, FsPermission actualPerm) {
-  Assert.assertEquals(expectedPerm.applyUMask(getFileMask()), actualPerm);
+    assertEquals(expectedPerm.applyUMask(getFileMask()), actualPerm);
   }