feat(csharp/test/Drivers/Databricks): Support token refresh to extend connection lifetime (#3177)

## Motivation

In scenarios like PowerBI dataset refresh, if a query runs longer than
the OAuth token's expiration time (typically 1 hour for AAD tokens), the
connection fails. PowerBI only refreshes access tokens if they have less
than 20 minutes of expiration time and never updates tokens after a
connection is opened.

This PR implements token refresh functionality in the Databricks ADBC
driver using the Databricks token exchange API. When an OAuth token is
about to expire within a configurable time limit, the driver
automatically exchanges it for a new token with a longer expiration
time.

## Key Components

1. **JWT Token Decoder**: Parses JWT tokens to extract expiration time
2. **Token Exchange Client**: Handles API calls to the Databricks token
exchange endpoint
3. **Token Exchange Handler**: HTTP handler that intercepts requests and
refreshes tokens when needed

## Changes

- Added new connection string parameter
`adbc.databricks.token_renew_limit` to control when token renewal
happens
- Implemented JWT token decoding to extract token expiration time
- Created token exchange client to handle API calls to Databricks token
exchange endpoint
- Added HTTP handler to intercept requests and refresh tokens when
needed
- Updated connection handling to create and configure the token exchange
components

## Testing

- Unit tests for JWT token decoding, token exchange client, and token
exchange handler
- End-to-end tests that verify token refresh functionality with real
tokens

```
dotnet test --filter "FullyQualifiedName~JwtTokenDecoderTests"
dotnet test --filter "FullyQualifiedName~TokenExchangeClientTests"
dotnet test --filter "FullyQualifiedName~TokenExchangeDelegatingHandlerTests"
```

Author: alexguo-db

csharp/src/Drivers/Databricks/Auth/JwtTokenDecoder.cs

@@ -0,0 +1,96 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Text;
+using System.Text.Json;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks.Auth
+{
+    /// <summary>
+    /// Utility class for decoding JWT tokens and extracting claims.
+    /// </summary>
+    internal static class JwtTokenDecoder
+    {
+        /// <summary>
+        /// Tries to parse a JWT token and extract its expiration time.
+        /// </summary>
+        /// <param name="token">The JWT token to parse.</param>
+        /// <param name="expiryTime">The extracted expiration time, if successful.</param>
+        /// <returns>True if the expiration time was successfully extracted, false otherwise.</returns>
+        public static bool TryGetExpirationTime(string token, out DateTime expiryTime)
+        {
+            expiryTime = DateTime.MinValue;
+
+            try
+            {
+                // JWT tokens have three parts separated by dots: header.payload.signature
+                string[] parts = token.Split('.');
+                if (parts.Length != 3)
+                {
+                    return false;
+                }
+
+                string payload = DecodeBase64Url(parts[1]);
+
+                using JsonDocument jsonDoc = JsonDocument.Parse(payload);
+
+                if (!jsonDoc.RootElement.TryGetProperty("exp", out JsonElement expElement))
+                {
+                    return false;
+                }
+
+                // The exp claim is a Unix timestamp (seconds since epoch)
+                if (!expElement.TryGetInt64(out long expSeconds))
+                {
+                    return false;
+                }
+
+                expiryTime = DateTimeOffset.FromUnixTimeSeconds(expSeconds).UtcDateTime;
+                return true;
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Decodes a base64url encoded string to a regular string.
+        /// </summary>
+        /// <param name="base64Url">The base64url encoded string.</param>
+        /// <returns>The decoded string.</returns>
+        private static string DecodeBase64Url(string base64Url)
+        {
+            // Convert base64url to base64
+            string base64 = base64Url
+                .Replace('-', '+')
+                .Replace('_', '/');
+
+            // Add padding if needed
+            switch (base64.Length % 4)
+            {
+                case 2: base64 += "=="; break;
+                case 3: base64 += "="; break;
+            }
+
+            byte[] bytes = Convert.FromBase64String(base64);
+
+            return Encoding.UTF8.GetString(bytes);
+        }
+    }
+}

csharp/src/Drivers/Databricks/Auth/OAuthClientCredentialsProvider.cs

@@ -228,7 +228,6 @@ public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToke
         public void Dispose()
         {
             _tokenLock.Dispose();
-            _httpClient.Dispose();
         }
 
         public string? GetCachedTokenScope()

csharp/src/Drivers/Databricks/Auth/TokenExchangeClient.cs

@@ -0,0 +1,177 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks.Auth
+{
+    /// <summary>
+    /// Response from the token exchange API.
+    /// </summary>
+    internal class TokenExchangeResponse
+    {
+        /// <summary>
+        /// The new access token.
+        /// </summary>
+        public string AccessToken { get; set; } = string.Empty;
+
+        /// <summary>
+        /// The token type (e.g., "Bearer").
+        /// </summary>
+        public string TokenType { get; set; } = string.Empty;
+
+        /// <summary>
+        /// The number of seconds until the token expires.
+        /// </summary>
+        public int ExpiresIn { get; set; }
+
+        /// <summary>
+        /// The calculated expiration time based on ExpiresIn.
+        /// </summary>
+        public DateTime ExpiryTime { get; set; }
+    }
+
+    /// <summary>
+    /// Interface for token exchange operations.
+    /// </summary>
+    internal interface ITokenExchangeClient
+    {
+        /// <summary>
+        /// Exchanges the provided token for a new token.
+        /// </summary>
+        /// <param name="token">The token to exchange.</param>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>The response from the token exchange API.</returns>
+        Task<TokenExchangeResponse> ExchangeTokenAsync(string token, CancellationToken cancellationToken);
+    }
+
+    /// <summary>
+    /// Client for exchanging tokens using the Databricks token exchange API.
+    /// </summary>
+    internal class TokenExchangeClient : ITokenExchangeClient
+    {
+        private readonly HttpClient _httpClient;
+        private readonly string _tokenExchangeEndpoint;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TokenExchangeClient"/> class.
+        /// </summary>
+        /// <param name="httpClient">The HTTP client to use for requests.</param>
+        /// <param name="host">The host of the Databricks workspace.</param>
+        public TokenExchangeClient(HttpClient httpClient, string host)
+        {
+            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+
+            if (string.IsNullOrEmpty(host))
+            {
+                throw new ArgumentNullException(nameof(host));
+            }
+
+            // Ensure the host doesn't have a trailing slash
+            host = host.TrimEnd('/');
+
+            _tokenExchangeEndpoint = $"https://{host}/oidc/v1/token";
+        }
+
+        /// <summary>
+        /// Exchanges the provided token for a new token.
+        /// </summary>
+        /// <param name="token">The token to exchange.</param>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>The response from the token exchange API.</returns>
+        public async Task<TokenExchangeResponse> ExchangeTokenAsync(string token, CancellationToken cancellationToken)
+        {
+            var content = new FormUrlEncodedContent(new[]
+            {
+                new KeyValuePair<string, string>("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
+                new KeyValuePair<string, string>("assertion", token)
+            });
+
+            var request = new HttpRequestMessage(HttpMethod.Post, _tokenExchangeEndpoint)
+            {
+                Content = content
+            };
+            request.Headers.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("*/*"));
+
+            HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken);
+
+            response.EnsureSuccessStatusCode();
+
+            string responseContent = await response.Content.ReadAsStringAsync();
+            return ParseTokenResponse(responseContent);
+        }
+
+        /// <summary>
+        /// Parses the token exchange API response.
+        /// </summary>
+        /// <param name="responseContent">The response content to parse.</param>
+        /// <returns>The parsed token exchange response.</returns>
+        private TokenExchangeResponse ParseTokenResponse(string responseContent)
+        {
+            using JsonDocument jsonDoc = JsonDocument.Parse(responseContent);
+            var root = jsonDoc.RootElement;
+
+            if (!root.TryGetProperty("access_token", out JsonElement accessTokenElement))
+            {
+                throw new DatabricksException("Token exchange response did not contain an access_token");
+            }
+
+            string? accessToken = accessTokenElement.GetString();
+            if (string.IsNullOrEmpty(accessToken))
+            {
+                throw new DatabricksException("Token exchange access_token was null or empty");
+            }
+
+            if (!root.TryGetProperty("token_type", out JsonElement tokenTypeElement))
+            {
+                throw new DatabricksException("Token exchange response did not contain token_type");
+            }
+
+            string? tokenType = tokenTypeElement.GetString();
+            if (string.IsNullOrEmpty(tokenType))
+            {
+                throw new DatabricksException("Token exchange token_type was null or empty");
+            }
+
+            if (!root.TryGetProperty("expires_in", out JsonElement expiresInElement))
+            {
+                throw new DatabricksException("Token exchange response did not contain expires_in");
+            }
+
+            int expiresIn = expiresInElement.GetInt32();
+            if (expiresIn <= 0)
+            {
+                throw new DatabricksException("Token exchange expires_in value must be positive");
+            }
+
+            DateTime expiryTime = DateTime.UtcNow.AddSeconds(expiresIn);
+
+            return new TokenExchangeResponse
+            {
+                AccessToken = accessToken!,
+                TokenType = tokenType!,
+                ExpiresIn = expiresIn,
+                ExpiryTime = expiryTime
+            };
+        }
+    }
+}