Merge branch 'trunk' of https://github.com/ABFSDriver/AbfsHadoop into HADOOP-19595

Author: bhattmanish98

dev-support/docker/Dockerfile_windows_10

@@ -62,7 +62,7 @@ RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.1
 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
 
 # Install Apache Maven.
-RUN powershell Invoke-WebRequest -URI https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
+RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.8-bin.zip -DestinationPath "C:\Maven"
 
 # Install CMake 3.19.0.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java

@@ -258,7 +258,14 @@ public Path(String scheme, String authority, String path) {
   private void initialize(String scheme, String authority, String path,
       String fragment) {
     try {
-      this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment)
+      // Normalize the path
+      String normalizedPath = normalizePath(scheme, path);
+
+      // Windows-specific fix: file URIs must start with "/"
+      if ("file".equalsIgnoreCase(scheme) && normalizedPath.matches("^[A-Za-z]:.*")) {
+        normalizedPath = "/" + normalizedPath.replace("\\", "/");
+      }
+      this.uri = new URI(scheme, authority, normalizedPath, null, fragment)
         .normalize();
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

@@ -128,6 +128,7 @@
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SEMICOLON;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SINGLE_WHITE_SPACE;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.UTF_8;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType.BLOB;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_IDENTITY_TRANSFORM_CLASS;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
@@ -153,6 +154,7 @@ public abstract class AbfsClient implements Closeable {
   public static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class);
   public static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON;
   public static final String ABFS_CLIENT_TIMER_THREAD_NAME = "abfs-timer-client";
+  public static final String FNS_BLOB_USER_AGENT_IDENTIFIER = "FNS";
 
   private final URL baseUrl;
   private final SharedKeyCredentials sharedKeyCredentials;
@@ -1319,6 +1321,14 @@ String initializeUserAgent(final AbfsConfiguration abfsConfiguration,
     sb.append(FORWARD_SLASH);
     sb.append(abfsConfiguration.getClusterType());
 
+    // Add a unique identifier in FNS-Blob user agent string
+    if (!getIsNamespaceEnabled()
+        && abfsConfiguration.getFsConfiguredServiceType() == BLOB) {
+      sb.append(SEMICOLON)
+          .append(SINGLE_WHITE_SPACE)
+          .append(FNS_BLOB_USER_AGENT_IDENTIFIER);
+    }
+
     sb.append(")");
 
     appendIfNotEmpty(sb, abfsConfiguration.getCustomUserAgentPrefix(), false);

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java

@@ -106,6 +106,7 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
 
   private static final String ACCOUNT_NAME = "bogusAccountName.dfs.core.windows.net";
   private static final String FS_AZURE_USER_AGENT_PREFIX = "Partner Service";
+  private static final String FNS_BLOB_USER_AGENT_IDENTIFIER = "FNS";
   private static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON;
   private static final String TEST_PATH = "/testfile";
   public static final int REDUCED_RETRY_COUNT = 2;
@@ -355,6 +356,42 @@ public void verifyUserAgentClusterType() throws Exception {
       .contains(DEFAULT_VALUE_UNKNOWN);
   }
 
+  @Test
+  // Test to verify the unique identifier in user agent string for FNS-Blob accounts
+  public void verifyUserAgentForFNSBlob() throws Exception {
+    assumeHnsDisabled();
+    assumeBlobServiceType();
+    final AzureBlobFileSystem fs = getFileSystem();
+    final AbfsConfiguration configuration = fs.getAbfsStore()
+        .getAbfsConfiguration();
+
+    String userAgentStr = getUserAgentString(configuration, false);
+    verifyBasicInfo(userAgentStr);
+    Assertions.assertThat(userAgentStr)
+        .describedAs(
+            "User-Agent string for FNS accounts on Blob endpoint should contain "
+                + FNS_BLOB_USER_AGENT_IDENTIFIER)
+        .contains(FNS_BLOB_USER_AGENT_IDENTIFIER);
+  }
+
+  @Test
+  // Test to verify that the user agent string for non-FNS-Blob accounts
+  // does not contain the FNS identifier.
+  public void verifyUserAgentForDFS() throws Exception {
+    assumeDfsServiceType();
+    final AzureBlobFileSystem fs = getFileSystem();
+    final AbfsConfiguration configuration = fs.getAbfsStore()
+        .getAbfsConfiguration();
+
+    String userAgentStr = getUserAgentString(configuration, false);
+    verifyBasicInfo(userAgentStr);
+    Assertions.assertThat(userAgentStr)
+        .describedAs(
+            "User-Agent string for non-FNS-Blob accounts should not contain"
+                + FNS_BLOB_USER_AGENT_IDENTIFIER)
+        .doesNotContain(FNS_BLOB_USER_AGENT_IDENTIFIER);
+  }
+
   public static AbfsClient createTestClientFromCurrentContext(
       AbfsClient baseAbfsClientInstance,
       AbfsConfiguration abfsConfig) throws IOException, URISyntaxException {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java

@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.io.PrintStream;
 import java.io.PrintWriter;
+import java.util.UUID;
 
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -43,8 +44,7 @@ public class FSConfigConverterTestCommons {
       new File(TEST_DIR, "test-yarn-site.xml").getAbsolutePath();
   public final static String CONVERSION_RULES_FILE =
       new File(TEST_DIR, "test-conversion-rules.properties").getAbsolutePath();
-  public final static String OUTPUT_DIR =
-      new File(TEST_DIR, "conversion-output").getAbsolutePath();
+  public static String OUTPUT_DIR;
 
   private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
   private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
@@ -57,7 +57,11 @@ public FSConfigConverterTestCommons() {
   }
 
   public void setUp() throws IOException {
-    File d = new File(TEST_DIR, "conversion-output");
+    String uuid = UUID.randomUUID().toString().trim();
+    OUTPUT_DIR =
+            new File(TEST_DIR, "conversion-output" + uuid).getAbsolutePath();
+    File d = new File(TEST_DIR, "conversion-output" + uuid);
+
     if (d.exists()) {
       FileUtils.deleteDirectory(d);
     }