Adding support for new storage serviceUri configurations (#19647)

* Adding support for new storage extensions configurations

* remove local artifact building

* Adding Configuration tests for Blob and Queue extensions

* fixing queue config tests

sdk/extensions/Microsoft.Azure.WebJobs.Extensions.Clients/src/Shared/StorageClientProvider.cs

@@ -0,0 +1,186 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Globalization;
+using Azure.Core;
+using Microsoft.Azure.WebJobs.Host;
+using Microsoft.Extensions.Azure;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Extensions.Clients.Shared
+{
+    /// <summary>
+    /// Abstraction to provide storage clients from the connection names.
+    /// This gets the storage account name via the binding attribute's <see cref="IConnectionProvider.Connection"/>
+    /// property.
+    /// If the connection is not specified on the attribute, it uses a default account.
+    /// </summary>
+    internal abstract class StorageClientProvider<TClient, TClientOptions> where TClientOptions : ClientOptions
+    {
+        private readonly IConfiguration _configuration;
+        private readonly AzureComponentFactory _componentFactory;
+        private readonly AzureEventSourceLogForwarder _logForwarder;
+        private readonly ILogger _logger;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StorageClientProvider{TClient, TClientOptions}"/> class that uses the registered Azure services.
+        /// </summary>
+        /// <param name="configuration">The configuration to use when creating Client-specific objects. <see cref="IConfiguration"/></param>
+        /// <param name="componentFactory">The Azure factory responsible for creating clients. <see cref="AzureComponentFactory"/></param>
+        /// <param name="logForwarder">Log forwarder that forwards events to ILogger. <see cref="AzureEventSourceLogForwarder"/></param>
+        /// <param name="logger">Logger used when there is an error creating a client</param>
+        public StorageClientProvider(IConfiguration configuration, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder, ILogger<TClient> logger)
+        {
+            _configuration = configuration;
+            _componentFactory = componentFactory;
+            _logForwarder = logForwarder;
+            _logger = logger;
+
+            _logForwarder?.Start();
+        }
+
+        /// <summary>
+        /// Gets the subdomain for the resource (i.e. blob, queue, file, table)
+        /// </summary>
+#pragma warning disable CA1056 // URI-like properties should not be strings
+        protected abstract string ServiceUriSubDomain { get; }
+#pragma warning restore CA1056 // URI-like properties should not be strings
+
+        /// <summary>
+        /// Gets the storage client specified by <paramref name="name"/>
+        /// </summary>
+        /// <param name="name">Name of the connection to use</param>
+        /// <param name="resolver">A resolver to interpret the provided connection <paramref name="name"/>.</param>
+        /// <returns>Client that was created.</returns>
+        public virtual TClient Get(string name, INameResolver resolver)
+        {
+            var resolvedName = resolver.ResolveWholeString(name);
+            return this.Get(resolvedName);
+        }
+
+        /// <summary>
+        /// Gets the storage client specified by <paramref name="name"/>
+        /// </summary>
+        /// <param name="name">Name of the connection to use</param>
+        /// <returns>Client that was created.</returns>
+        public virtual TClient Get(string name)
+        {
+            if (string.IsNullOrWhiteSpace(name))
+            {
+                name = ConnectionStringNames.Storage; // default
+            }
+
+            // $$$ Where does validation happen?
+            IConfigurationSection connectionSection = _configuration.GetWebJobsConnectionStringSection(name);
+            if (!connectionSection.Exists())
+            {
+                // Not found
+                throw new InvalidOperationException($"Storage account connection string '{IConfigurationExtensions.GetPrefixedConnectionStringName(name)}' does not exist. Make sure that it is a defined App Setting.");
+            }
+
+            var credential = _componentFactory.CreateTokenCredential(connectionSection);
+            var options = CreateClientOptions(connectionSection);
+            return CreateClient(connectionSection, credential, options);
+        }
+
+        /// <summary>
+        /// Creates a storage client
+        /// </summary>
+        /// <param name="configuration">The <see cref="IConfiguration"/> to use when creating Client-specific objects.</param>
+        /// <param name="tokenCredential">The <see cref="TokenCredential"/> to authenticate for requests.</param>
+        /// <param name="options">Generic options to use for the client</param>
+        /// <returns>Storage client</returns>
+        protected virtual TClient CreateClient(IConfiguration configuration, TokenCredential tokenCredential, TClientOptions options)
+        {
+            // If connection string is present, it will be honored first
+            if (!IsConnectionStringPresent(configuration) && TryGetServiceUri(configuration, out Uri serviceUri))
+            {
+                var constructor = typeof(TClient).GetConstructor(new Type[] { typeof(Uri), typeof(TokenCredential), typeof(TClientOptions) });
+                return (TClient)constructor.Invoke(new object[] { serviceUri, tokenCredential, options });
+            }
+
+            return (TClient)_componentFactory.CreateClient(typeof(TClient), configuration, tokenCredential, options);
+        }
+
+        /// <summary>
+        /// The host account is for internal storage mechanisms like load balancer queuing.
+        /// </summary>
+        /// <returns>Storage client</returns>
+        public virtual TClient GetHost()
+        {
+            return this.Get(null);
+        }
+
+        /// <summary>
+        /// Creates client options from the given configuration
+        /// </summary>
+        /// <param name="configuration">Registered <see cref="IConfiguration"/></param>
+        /// <returns>Client options</returns>
+        protected virtual TClientOptions CreateClientOptions(IConfiguration configuration)
+        {
+            var clientOptions = (TClientOptions)_componentFactory.CreateClientOptions(typeof(TClientOptions), null, configuration);
+            return clientOptions;
+        }
+
+        /// <summary>
+        /// Either constructs the serviceUri from the provided accountName
+        /// or retrieves the serviceUri for the specific resource (i.e. blobServiceUri or queueServiceUri)
+        /// </summary>
+        /// <param name="configuration">Registered <see cref="IConfiguration"/></param>
+        /// <param name="serviceUri">instantiates the serviceUri</param>
+        /// <returns>retrieval success</returns>
+        protected virtual bool TryGetServiceUri(IConfiguration configuration, out Uri serviceUri)
+        {
+            try
+            {
+                var serviceUriConfig = string.Format(CultureInfo.InvariantCulture, "{0}ServiceUri", ServiceUriSubDomain);
+
+                string accountName;
+                string uriStr;
+                if ((accountName = configuration.GetValue<string>("accountName")) != null)
+                {
+                    serviceUri = FormatServiceUri(accountName);
+                    return true;
+                }
+                else if ((uriStr = configuration.GetValue<string>(serviceUriConfig)) != null)
+                {
+                    serviceUri = new Uri(uriStr);
+                    return true;
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Could not parse serviceUri from the configuration.");
+            }
+
+            serviceUri = default(Uri);
+            return false;
+        }
+
+        /// <summary>
+        /// Generates the serviceUri for a particular storage resource
+        /// </summary>
+        /// <param name="accountName">accountName for the storage account</param>
+        /// <param name="defaultProtocol">protocol to use for REST requests</param>
+        /// <param name="endpointSuffix">endpoint suffix for the storage account</param>
+        /// <returns>Uri for the storage resource</returns>
+        protected virtual Uri FormatServiceUri(string accountName, string defaultProtocol = "https", string endpointSuffix = "core.windows.net")
+        {
+            // Todo: Eventually move this into storage sdk
+            var uri = string.Format(CultureInfo.InvariantCulture, "{0}://{1}.{2}.{3}", defaultProtocol, accountName, ServiceUriSubDomain, endpointSuffix);
+            return new Uri(uri);
+        }
+
+        /// <summary>
+        /// Checks if the specified <see cref="IConfiguration"/> object represents a connection string.
+        /// </summary>
+        /// <param name="configuration">The <see cref="IConfiguration"/> to check</param>
+        /// <returns>true if this <see cref="IConfiguration"/> object is a connection string; false otherwise.</returns>
+        protected static bool IsConnectionStringPresent(IConfiguration configuration)
+        {
+            return configuration is IConfigurationSection section && section.Value != null;
+        }
+    }
+}

sdk/storage/Azure.Storage.Common/tests/Shared/AzuriteFixture.cs

@@ -230,7 +230,8 @@ public class AzuriteAccount
         public int BlobsPort { get; set; }
         public int QueuesPort { get; set; }
 
-        public string Endpoint => $"https://127.0.0.1:{BlobsPort}/{Name}";
+        public string BlobEndpoint => $"https://127.0.0.1:{BlobsPort}/{Name}";
+        public string QueueEndpoint => $"https://127.0.0.1:{QueuesPort}/{Name}";
 
         public string ConnectionString
         {

sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/BlobServiceClientProvider.cs

@@ -4,15 +4,25 @@
 using System;
 using Azure.Core;
 using Azure.Storage.Blobs;
-using Microsoft.Azure.WebJobs.Extensions.Storage.Common;
+using Microsoft.Azure.WebJobs.Extensions.Clients.Shared;
 using Microsoft.Extensions.Azure;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
 
 namespace Microsoft.Azure.WebJobs.Extensions.Storage.Blobs
 {
     internal class BlobServiceClientProvider : StorageClientProvider<BlobServiceClient, BlobClientOptions>
     {
-        public BlobServiceClientProvider(IConfiguration configuration, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder)
-            : base(configuration, componentFactory, logForwarder) {}
+        public BlobServiceClientProvider(IConfiguration configuration, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder, ILogger<BlobServiceClient> logger)
+            : base(configuration, componentFactory, logForwarder, logger) { }
+
+        /// <inheritdoc/>
+        protected override string ServiceUriSubDomain
+        {
+            get
+            {
+                return "blob";
+            }
+        }
     }
-}
+}

sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/src/QueueServiceClientProvider.cs

@@ -5,8 +5,7 @@
 using System.Threading.Tasks;
 using Azure.Core;
 using Azure.Storage.Queues;
-using Azure.Storage.Queues.Models;
-using Microsoft.Azure.WebJobs.Extensions.Storage.Common;
+using Microsoft.Azure.WebJobs.Extensions.Clients.Shared;
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Extensions.Azure;
 using Microsoft.Extensions.Configuration;
@@ -18,20 +17,29 @@ namespace Microsoft.Azure.WebJobs.Extensions.Storage.Blobs
     internal class QueueServiceClientProvider : StorageClientProvider<QueueServiceClient, QueueClientOptions>
     {
         private readonly QueuesOptions _queuesOptions;
-        private readonly ILogger<QueueServiceClientProvider> _logger;
+        private readonly ILogger<QueueServiceClient> _logger;
 
         public QueueServiceClientProvider(
             IConfiguration configuration,
             AzureComponentFactory componentFactory,
             AzureEventSourceLogForwarder logForwarder,
             IOptions<QueuesOptions> queueOptions,
-            ILogger<QueueServiceClientProvider> logger)
-            : base(configuration, componentFactory, logForwarder)
+            ILogger<QueueServiceClient> logger)
+            : base(configuration, componentFactory, logForwarder, logger)
         {
             _queuesOptions = queueOptions?.Value;
             _logger = logger;
         }
 
+        /// <inheritdoc/>
+        protected override string ServiceUriSubDomain
+        {
+            get
+            {
+                return "queue";
+            }
+        }
+
         protected override QueueClientOptions CreateClientOptions(IConfiguration configuration)
         {
             var options = base.CreateClientOptions(configuration);

sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/BlobConfigurationTests.cs

@@ -66,7 +66,41 @@ public async Task BlobClient_CanConnect_ServiceUri()
                 {
                     cb.AddInMemoryCollection(new Dictionary<string, string>()
                     {
-                        {"CustomConnection:serviceUri", account.Endpoint },
+                        {"CustomConnection:serviceUri", account.BlobEndpoint },
+                        {"blobPath", "endpointcontainer/endpointblob" }
+                    });
+                })
+                .ConfigureDefaultTestHost(prog, builder =>
+                {
+                    SetupAzurite(builder);
+                    builder.AddAzureStorageBlobs();
+                })
+                .Build();
+
+            var jobHost = host.GetJobHost<BindToCloudBlockBlobProgram>();
+            await jobHost.CallAsync(nameof(BindToCloudBlockBlobProgram.Run));
+
+            var result = prog.Result;
+
+            // Assert
+            Assert.NotNull(result);
+            Assert.AreEqual("endpointblob", result.Name);
+            Assert.AreEqual("endpointcontainer", result.BlobContainerName);
+            Assert.NotNull(result.BlobContainerName);
+            Assert.False(await result.ExistsAsync());
+        }
+
+        [Test]
+        public async Task BlobClient_CanConnect_BlobServiceUri()
+        {
+            var account = azuriteFixture.GetAzureAccount();
+            var prog = new BindToCloudBlockBlobProgram();
+            IHost host = new HostBuilder()
+                .ConfigureAppConfiguration(cb =>
+                {
+                    cb.AddInMemoryCollection(new Dictionary<string, string>()
+                    {
+                        {"CustomConnection:blobServiceUri", account.BlobEndpoint },
                         {"blobPath", "endpointcontainer/endpointblob" }
                     });
                 })

sdk/storage/Microsoft.Azure.WebJobs.Extensions.Storage.Blobs/tests/FakeBlobServiceClientProvider.cs

@@ -12,7 +12,7 @@ internal class FakeBlobServiceClientProvider : BlobServiceClientProvider
         private readonly BlobServiceClient _blobServiceClient;
 
         public FakeBlobServiceClientProvider(BlobServiceClient blobServiceClient)
-            : base(null, null, null) {
+            : base(null, null, null, null) {
             _blobServiceClient = blobServiceClient;
         }