HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled" to be set as Account Specific Config (#7062)

Author: anujmodi2021

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -414,7 +414,8 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName)
   }
 
   public Trilean getIsNamespaceEnabledAccount() {
-    return Trilean.getTrilean(isNamespaceEnabledAccount);
+    return Trilean.getTrilean(
+        getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount));
   }
 
   /**

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java

@@ -40,6 +40,7 @@
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -259,6 +260,9 @@ public void testHttpReadTimeout() throws Exception {
 
   public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs)
       throws Exception {
+    // This is to make sure File System creation goes through before network calls start failing.
+    assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+
     Configuration conf = this.getRawConfiguration();
     // set to small values that will cause timeouts
     conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs);

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java

@@ -40,6 +40,8 @@
 import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doReturn;
@@ -125,6 +127,8 @@ private AzureBlobFileSystem getNewFSWithHnsConf(
     Configuration rawConfig = new Configuration();
     rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
     rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount);
+    rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
+        this.getAccountName()), isNamespaceEnabledAccount);
     rawConfig
         .setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
     rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
@@ -265,4 +269,65 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
     Mockito.verify(mockClient, times(1))
         .getAclStatus(anyString(), any(TracingContext.class));
   }
+
+  @Test
+  public void testAccountSpecificConfig() throws Exception {
+    Configuration rawConfig = new Configuration();
+    rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME);
+    rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+    rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED,
+        this.getAccountName()));
+    String testAccountName = "testAccount.dfs.core.windows.net";
+    String otherAccountName = "otherAccount.dfs.core.windows.net";
+    String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
+    String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
+
+    // Set both account specific and account agnostic config for test account
+    rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
+    rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
+    // Assert that account specific config takes precedence
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+    assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+    // Assert that other account still uses account agnostic config
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, otherUri);
+    assertFileSystemInitWithExpectedHNSSettings(rawConfig, true);
+
+    // Set only the account specific config for test account
+    rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
+    rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+    // Assert that only account specific config is enough for test account
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+    assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+
+    // Set only account agnostic config
+    rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR);
+    rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+    assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+
+    // Unset both account specific and account agnostic config
+    rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED);
+    rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName));
+    rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
+    rawConfig.set(AZURE_MAX_IO_RETRIES, "0");
+    // Assert that file system init fails with UnknownHost exception as getAcl() is needed.
+    try {
+      assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);
+    } catch (Exception e) {
+      Assertions.assertThat(e.getCause().getMessage())
+          .describedAs("getAcl() to determine HNS Nature of account should"
+              + "fail with Unknown Host Exception").contains("UnknownHostException");
+    }
+  }
+
+  private void assertFileSystemInitWithExpectedHNSSettings(
+      Configuration configuration, boolean expectedIsHnsEnabledValue) throws IOException {
+    try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration)) {
+      Assertions.assertThat(getIsNamespaceEnabled(fs)).describedAs(
+          "getIsNamespaceEnabled should return true when the "
+              + "account specific config is not set").isEqualTo(expectedIsHnsEnabledValue);
+    } catch (Exception e) {
+      throw e;
+    }
+  }
 }