HADOOP-19736: ABFS. Support for new auth type: User-bound SAS

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -1279,7 +1279,7 @@ public boolean shouldTrackLatency() {
 
   public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException {
     AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey);
-    if (authType == AuthType.OAuth) {
+    if (authType == AuthType.OAuth || authType == AuthType.UserboundSASWithOAuth) {
       try {
         Class<? extends AccessTokenProvider> tokenProviderClass =
             getTokenProviderClass(authType,
@@ -1474,6 +1474,49 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio
     }
   }
 
+  /**
+   * Returns the SASTokenProvider implementation to be used to generate user-bound SAS token.<br>
+   * Custom implementation of {@link SASTokenProvider} under th config
+   * "fs.azure.sas.token.provider.type" needs to be provided.<br>
+   * @return sasTokenProvider object based on configurations provided
+   * @throws AzureBlobFileSystemException
+   */
+  public SASTokenProvider getSASTokenProviderForUserBoundSAS() throws AzureBlobFileSystemException {
+    AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey);
+    if (authType != AuthType.UserboundSASWithOAuth) {
+      throw new SASTokenProviderException(String.format(
+          "Invalid auth type: %s is being used, expecting user-bound SAS.", authType));
+    }
+
+    try {
+      Class<? extends SASTokenProvider> customSasTokenProviderImplementation =
+          getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE,
+              null, SASTokenProvider.class);
+
+      if (customSasTokenProviderImplementation == null) {
+        throw new SASTokenProviderException(String.format(
+            "\"%s\" must be set for user-bound SAS auth type.",
+            FS_AZURE_SAS_TOKEN_PROVIDER_TYPE));
+      }
+
+        SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(
+            customSasTokenProviderImplementation, rawConfig);
+        if (sasTokenProvider == null) {
+          throw new SASTokenProviderException(String.format(
+              "Failed to initialize %s", customSasTokenProviderImplementation));
+        }
+        LOG.trace("Initializing {}", customSasTokenProviderImplementation.getName());
+        sasTokenProvider.initialize(rawConfig, accountName);
+        LOG.trace("{} init complete", customSasTokenProviderImplementation.getName());
+        return sasTokenProvider;
+    } catch (SASTokenProviderException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new SASTokenProviderException(
+          "Unable to load user-bound SAS token provider class: " + e, e);
+    }
+  }
+
   public EncryptionContextProvider createEncryptionContextProvider() {
     try {
       String configKey = FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE;

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java

@@ -1741,7 +1741,15 @@ private void initializeClient(URI uri, String fileSystemName,
     } else if (authType == AuthType.SAS) {
       LOG.trace("Fetching SAS Token Provider");
       sasTokenProvider = abfsConfiguration.getSASTokenProvider();
-    } else {
+    } else if(authType == AuthType.UserboundSASWithOAuth){
+      LOG.trace("Fetching SAS and OAuth Token Provider for user bound SAS");
+      AzureADAuthenticator.init(abfsConfiguration);
+      tokenProvider = abfsConfiguration.getTokenProvider();
+      ExtensionHelper.bind(tokenProvider, uri,
+          abfsConfiguration.getRawConfiguration());
+      sasTokenProvider = abfsConfiguration.getSASTokenProviderForUserBoundSAS();
+    }
+    else {
       LOG.trace("Fetching token provider");
       tokenProvider = abfsConfiguration.getTokenProvider();
       ExtensionHelper.bind(tokenProvider, uri,
@@ -1770,7 +1778,12 @@ private void initializeClient(URI uri, String fileSystemName,
     }
 
     LOG.trace("Initializing AbfsClient for {}", baseUrl);
-    if (tokenProvider != null) {
+    if(tokenProvider != null && sasTokenProvider != null){
+      this.clientHandler = new AbfsClientHandler(baseUrl, creds, abfsConfiguration,
+          tokenProvider, sasTokenProvider, encryptionContextProvider,
+          populateAbfsClientContext());
+    }
+    else if (tokenProvider != null) {
       this.clientHandler = new AbfsClientHandler(baseUrl, creds, abfsConfiguration,
           tokenProvider, encryptionContextProvider,
           populateAbfsClientContext());

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java

@@ -187,7 +187,8 @@ public enum ApiVersion {
     DEC_12_2019("2019-12-12"),
     APR_10_2021("2021-04-10"),
     AUG_03_2023("2023-08-03"),
-    NOV_04_2024("2024-11-04");
+    NOV_04_2024("2024-11-04"),
+    JULY_05_2025("2025-07-05");
 
     private final String xMsApiVersion;
 
@@ -201,7 +202,7 @@ public String toString() {
     }
 
     public static ApiVersion getCurrentVersion() {
-      return NOV_04_2024;
+      return JULY_05_2025;
     }
   }
 

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

@@ -363,6 +363,21 @@ public AbfsClient(final URL baseUrl,
     this.sasTokenProvider = sasTokenProvider;
   }
 
+  public AbfsClient(final URL baseUrl,
+      final SharedKeyCredentials sharedKeyCredentials,
+      final AbfsConfiguration abfsConfiguration,
+      final AccessTokenProvider tokenProvider,
+      final SASTokenProvider sasTokenProvider,
+      final EncryptionContextProvider encryptionContextProvider,
+      final AbfsClientContext abfsClientContext,
+      final AbfsServiceType abfsServiceType)
+      throws IOException {
+    this(baseUrl, sharedKeyCredentials, abfsConfiguration,
+        encryptionContextProvider, abfsClientContext, abfsServiceType);
+    this.sasTokenProvider = sasTokenProvider;
+    this.tokenProvider = tokenProvider;
+  }
+
   @Override
   public void close() throws IOException {
     if (isMetricCollectionEnabled && runningTimerTask != null) {
@@ -1157,7 +1172,7 @@ protected String appendSASTokenToQuery(String path,
                                          String cachedSasToken)
       throws SASTokenProviderException {
     String sasToken = null;
-    if (this.authType == AuthType.SAS) {
+    if (this.authType == AuthType.SAS || this.authType == AuthType.UserboundSASWithOAuth) {
       try {
         LOG.trace("Fetch SAS token for {} on {}", operation, path);
         if (cachedSasToken == null) {

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientHandler.java

@@ -80,6 +80,25 @@ public AbfsClientHandler(final URL baseUrl,
         abfsClientContext);
   }
 
+  public AbfsClientHandler(final URL baseUrl,
+      final SharedKeyCredentials sharedKeyCredentials,
+      final AbfsConfiguration abfsConfiguration,
+      final AccessTokenProvider tokenProvider,
+      final SASTokenProvider sasTokenProvider,
+      final EncryptionContextProvider encryptionContextProvider,
+      final AbfsClientContext abfsClientContext) throws IOException {
+    // This will initialize the default and ingress service types.
+    // This is needed before creating the clients so that we can do cache warmup
+    // only for default client.
+    initServiceType(abfsConfiguration);
+    this.dfsAbfsClient = createDfsClient(baseUrl, sharedKeyCredentials,
+        abfsConfiguration, tokenProvider, sasTokenProvider, encryptionContextProvider,
+        abfsClientContext);
+    this.blobAbfsClient = createBlobClient(baseUrl, sharedKeyCredentials,
+        abfsConfiguration, tokenProvider, sasTokenProvider, encryptionContextProvider,
+        abfsClientContext);
+  }
+
   /**
    * Initialize the default service type based on the user configuration.
    * @param abfsConfiguration set by user.
@@ -154,7 +173,15 @@ private AbfsDfsClient createDfsClient(final URL baseUrl,
       final EncryptionContextProvider encryptionContextProvider,
       final AbfsClientContext abfsClientContext) throws IOException {
     URL dfsUrl = changeUrlFromBlobToDfs(baseUrl);
-    if (tokenProvider != null) {
+    if (tokenProvider != null && sasTokenProvider != null) {
+      LOG.debug(
+          "Creating AbfsDfsClient with both access token provider and SAS token provider using the URL: {}",
+          dfsUrl);
+      return new AbfsDfsClient(dfsUrl, creds, abfsConfiguration,
+          tokenProvider, sasTokenProvider, encryptionContextProvider,
+          abfsClientContext);
+    }
+    else if (tokenProvider != null) {
       LOG.debug("Creating AbfsDfsClient with access token provider using the URL: {}", dfsUrl);
       return new AbfsDfsClient(dfsUrl, creds, abfsConfiguration,
           tokenProvider, encryptionContextProvider,

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java

@@ -174,6 +174,17 @@ public AbfsDfsClient(final URL baseUrl,
         encryptionContextProvider, abfsClientContext, AbfsServiceType.DFS);
   }
 
+  public AbfsDfsClient(final URL baseUrl,
+      final SharedKeyCredentials sharedKeyCredentials,
+      final AbfsConfiguration abfsConfiguration,
+      final AccessTokenProvider tokenProvider,
+      final SASTokenProvider sasTokenProvider,
+      final EncryptionContextProvider encryptionContextProvider,
+      final AbfsClientContext abfsClientContext) throws IOException {
+    super(baseUrl, sharedKeyCredentials, abfsConfiguration, tokenProvider, sasTokenProvider,
+        encryptionContextProvider, abfsClientContext, AbfsServiceType.DFS);
+  }
+
   /**
    * Create request headers for Rest Operation using the default API version.
    * @return default request headers.

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java

@@ -570,6 +570,11 @@ public void signRequest(final AbfsHttpOperation httpOperation, int bytesToSign)
         // do nothing; the SAS token should already be appended to the query string
         httpOperation.setMaskForSAS(); //mask sig/oid from url for logs
         break;
+      case UserboundSASWithOAuth:
+        httpOperation.setRequestProperty(HttpHeaderConfigurations.AUTHORIZATION,
+            client.getAccessToken());
+        httpOperation.setMaskForSAS(); //mask sig/oid from url for logs
+        break;
       case SharedKey:
       default:
         // sign the HTTP request

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AuthType.java

@@ -24,5 +24,6 @@ public enum AuthType {
     SharedKey,
     OAuth,
     Custom,
-    SAS
+    SAS,
+     UserboundSASWithOAuth
 }

hadoop-tools/hadoop-azure/src/site/markdown/index.md

@@ -303,6 +303,7 @@ driven by them.
 3. Deployed in-Azure with the Azure VMs providing OAuth 2.0 tokens to the application, "Managed Instance".
 4. Using Shared Access Signature (SAS) tokens provided by a custom implementation of the SASTokenProvider interface.
 5. By directly configuring a fixed Shared Access Signature (SAS) token in the account configuration settings files.
+6. Using user-bound SAS auth type, which is requires OAuth 2.0 setup (point 2 above) and SAS setup (point 4 above)
 
 Note: SAS Based Authentication should be used only with HNS Enabled accounts.
 
@@ -783,6 +784,45 @@ requests. User can specify them as fixed SAS Token to be used across all the req
     - fs.azure.sas.fixed.token.ACCOUNT_NAME
     - fs.azure.sas.fixed.token
 
+### User-bound user delegation SAS 
+- **Description**: The user-bound SAS auth type allows to track the usage of the SAS token generated- something 
+ that was not possible in user-delegation SAS authentication type. Reach out to us at 'askabfs@microsoft.com' for more information.
+ To use this authentication type, both custom SAS token provider class (that implements org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider) as
+    well as OAuth 2.0 provider type need to be specified.
+    - Refer to 'Shared Access Signature (SAS) Token Provider' section above for user-delegation SAS token provider class details and example class implementation.
+    - There are multiple identity configurations for OAuth settings. Listing the main ones below:
+        - Client Credentials
+        - Custom token provider
+        - Managed Identity
+        - Workload Identity
+        Refer to respective OAuth 2.0 sections above to correctly chose the OAuth provider type
+
+
+- **Configuration**: To use this method with ABFS Driver, specify the following properties in your `core-site.xml` file:
+
+    1. Authentication Type:
+        ```xml
+        <property>
+          <name>fs.azure.account.auth.type</name>
+          <value>UserboundSASWithOAuth</value>
+        </property>
+        ```
+    2. OAuth 2.0 Provider Type:
+        ```xml
+        <property>
+          <name>fs.azure.account.oauth.provider.type</name>
+          <value>org.apache.hadoop.fs.azurebfs.oauth2.ADD_CHOSEN_OAUTH_IDENTITY_CONFIGURATION</value>
+        </property>
+       ```
+    3. Custom SAS Token Provider Class:
+        ```xml
+        <property>
+          <name>fs.azure.sas.token.provider.type</name>
+          <value>CUSTOM_SAS_TOKEN_PROVIDER_CLASS</value>
+        </property>
+        ```
+
+
 ## <a name="technical"></a> Technical notes
 
 ### <a name="proxy"></a> Proxy setup