From 2ea9632420822845fd874cc90e6c2d0ff0a75f14 Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:01:45 -0700 Subject: [PATCH 01/22] Give the users the ability to have control over ConfigurationClient instance(s) used by the provider (#598) * Introduced a new `AzureAppConfigurationClientFactory` class to handle the creation of `ConfigurationClient` instances * remove clients dictionary since we will not have hits and clients are already stored in ConfigurationClientManager * revert * add license + remove unused usings * ran dotnet format * add capability of fallback to different stores * add explicit type * address comments * remove scheme validation --------- Co-authored-by: Sami Sadfa --- .../AzureAppConfigurationClientFactory.cs | 74 +++++++++++++++++++ .../AzureAppConfigurationOptions.cs | 17 +++++ .../AzureAppConfigurationSource.cs | 32 ++++---- .../ConfigurationClientManager.cs | 64 +++------------- ...Configuration.AzureAppConfiguration.csproj | 1 + .../FailoverTests.cs | 19 +++-- 6 files changed, 128 insertions(+), 79 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs new file mode 100644 index 00000000..6127822d --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Core; +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal class AzureAppConfigurationClientFactory : IAzureClientFactory + { + private readonly ConfigurationClientOptions _clientOptions; + + private readonly TokenCredential _credential; + private readonly IEnumerable _connectionStrings; + + public AzureAppConfigurationClientFactory( + IEnumerable connectionStrings, + ConfigurationClientOptions clientOptions) + { + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + _connectionStrings = connectionStrings; + + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public AzureAppConfigurationClientFactory( + TokenCredential credential, + ConfigurationClientOptions clientOptions) + { + _credential = credential ?? throw new ArgumentNullException(nameof(credential)); + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public ConfigurationClient CreateClient(string endpoint) + { + if (string.IsNullOrEmpty(endpoint)) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uriResult)) + { + throw new ArgumentException("Invalid host URI."); + } + + if (_credential != null) + { + return new ConfigurationClient(uriResult, _credential, _clientOptions); + } + + string connectionString = _connectionStrings.FirstOrDefault(cs => ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection) == endpoint); + + // + // falback to the first connection string + if (connectionString == null) + { + string id = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.IdSection); + string secret = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.SecretSection); + + connectionString = ConnectionStringUtils.Build(uriResult, id, secret); + } + + return new ConfigurationClient(connectionString, _clientOptions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7d9a9cad..0c80b200 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -3,6 +3,7 @@ // using Azure.Core; using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; @@ -131,6 +132,11 @@ internal IEnumerable Adapters /// internal StartupOptions Startup { get; set; } = new StartupOptions(); + /// + /// Client factory that is responsible for creating instances of ConfigurationClient. + /// + internal IAzureClientFactory ClientFactory { get; private set; } + /// /// Initializes a new instance of the class. /// @@ -144,6 +150,17 @@ public AzureAppConfigurationOptions() }; } + /// + /// Sets the client factory used to create ConfigurationClient instances. + /// + /// The client factory. + /// The current instance. + public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) + { + ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); + return this; + } + /// /// Specify what key-values to include in the configuration provider. /// can be called multiple times to include multiple sets of key-values. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs index dee62006..83d20e2f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using System; +using System.Collections.Generic; +using System.Linq; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -29,35 +33,33 @@ public IConfigurationProvider Build(IConfigurationBuilder builder) try { AzureAppConfigurationOptions options = _optionsProvider(); - IConfigurationClientManager clientManager; if (options.ClientManager != null) { - clientManager = options.ClientManager; + return new AzureAppConfigurationProvider(options.ClientManager, options, _optional); } - else if (options.ConnectionStrings != null) + + IEnumerable endpoints; + IAzureClientFactory clientFactory = options.ClientFactory; + + if (options.ConnectionStrings != null) { - clientManager = new ConfigurationClientManager( - options.ConnectionStrings, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.ConnectionStrings.Select(cs => new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection))); + + clientFactory ??= new AzureAppConfigurationClientFactory(options.ConnectionStrings, options.ClientOptions); } else if (options.Endpoints != null && options.Credential != null) { - clientManager = new ConfigurationClientManager( - options.Endpoints, - options.Credential, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.Endpoints; + + clientFactory ??= new AzureAppConfigurationClientFactory(options.Credential, options.ClientOptions); } else { throw new ArgumentException($"Please call {nameof(AzureAppConfigurationOptions)}.{nameof(AzureAppConfigurationOptions.Connect)} to specify how to connect to Azure App Configuration."); } - provider = new AzureAppConfigurationProvider(clientManager, options, _optional); + provider = new AzureAppConfigurationProvider(new ConfigurationClientManager(clientFactory, endpoints, options.ReplicaDiscoveryEnabled, options.LoadBalancingEnabled), options, _optional); } catch (InvalidOperationException ex) // InvalidOperationException is thrown when any problems are found while configuring AzureAppConfigurationOptions or when SDK fails to create a configurationClient. { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs index a0215ca3..61840d03 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs @@ -2,10 +2,10 @@ // Licensed under the MIT license. // -using Azure.Core; using Azure.Data.AppConfiguration; using DnsClient; using DnsClient.Protocol; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; @@ -26,12 +26,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration /// internal class ConfigurationClientManager : IConfigurationClientManager, IDisposable { + private readonly IAzureClientFactory _clientFactory; private readonly IList _clients; + private readonly Uri _endpoint; - private readonly string _secret; - private readonly string _id; - private readonly TokenCredential _credential; - private readonly ConfigurationClientOptions _clientOptions; + private readonly bool _replicaDiscoveryEnabled; private readonly SrvLookupClient _srvLookupClient; private readonly string _validDomain; @@ -52,61 +51,20 @@ internal class ConfigurationClientManager : IConfigurationClientManager, IDispos internal int RefreshClientsCalled { get; set; } = 0; public ConfigurationClientManager( - IEnumerable connectionStrings, - ConfigurationClientOptions clientOptions, - bool replicaDiscoveryEnabled, - bool loadBalancingEnabled) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - string connectionString = connectionStrings.First(); - _endpoint = new Uri(ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.EndpointSection)); - _secret = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.SecretSection); - _id = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.IdSection); - _clientOptions = clientOptions; - _replicaDiscoveryEnabled = replicaDiscoveryEnabled; - - // If load balancing is enabled, shuffle the passed in connection strings to randomize the endpoint used on startup - if (loadBalancingEnabled) - { - connectionStrings = connectionStrings.ToList().Shuffle(); - } - - _validDomain = GetValidDomain(_endpoint); - _srvLookupClient = new SrvLookupClient(); - - _clients = connectionStrings - .Select(cs => - { - var endpoint = new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection)); - return new ConfigurationClientWrapper(endpoint, new ConfigurationClient(cs, _clientOptions)); - }) - .ToList(); - } - - public ConfigurationClientManager( + IAzureClientFactory clientFactory, IEnumerable endpoints, - TokenCredential credential, - ConfigurationClientOptions clientOptions, bool replicaDiscoveryEnabled, bool loadBalancingEnabled) { + _clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory)); + if (endpoints == null || !endpoints.Any()) { throw new ArgumentNullException(nameof(endpoints)); } - if (credential == null) - { - throw new ArgumentNullException(nameof(credential)); - } - _endpoint = endpoints.First(); - _credential = credential; - _clientOptions = clientOptions; + _replicaDiscoveryEnabled = replicaDiscoveryEnabled; // If load balancing is enabled, shuffle the passed in endpoints to randomize the endpoint used on startup @@ -119,7 +77,7 @@ public ConfigurationClientManager( _srvLookupClient = new SrvLookupClient(); _clients = endpoints - .Select(endpoint => new ConfigurationClientWrapper(endpoint, new ConfigurationClient(endpoint, _credential, _clientOptions))) + .Select(endpoint => new ConfigurationClientWrapper(endpoint, clientFactory.CreateClient(endpoint.AbsoluteUri))) .ToList(); } @@ -289,9 +247,7 @@ private async Task RefreshFallbackClients(CancellationToken cancellationToken) { var targetEndpoint = new Uri($"https://{host}"); - var configClient = _credential == null - ? new ConfigurationClient(ConnectionStringUtils.Build(targetEndpoint, _id, _secret), _clientOptions) - : new ConfigurationClient(targetEndpoint, _credential, _clientOptions); + ConfigurationClient configClient = _clientFactory.CreateClient(targetEndpoint.AbsoluteUri); newDynamicClients.Add(new ConfigurationClientWrapper(targetEndpoint, configClient)); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 3997c788..dc844599 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -19,6 +19,7 @@ + diff --git a/tests/Tests.AzureAppConfiguration/FailoverTests.cs b/tests/Tests.AzureAppConfiguration/FailoverTests.cs index 86ea96b9..929f9bef 100644 --- a/tests/Tests.AzureAppConfiguration/FailoverTests.cs +++ b/tests/Tests.AzureAppConfiguration/FailoverTests.cs @@ -268,10 +268,11 @@ public void FailOverTests_AutoFailover() [Fact] public void FailOverTests_ValidateEndpoints() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -285,9 +286,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager.IsValidEndpoint("azure.azconfig.bad.io")); var configClientManager2 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.appconfig.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -301,9 +301,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager2.IsValidEndpoint("azure.appconfigbad.azure.com")); var configClientManager3 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig-test.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -311,9 +310,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager3.IsValidEndpoint("azure.azconfig.io")); var configClientManager4 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.z1.appconfig-test.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -325,10 +323,11 @@ public void FailOverTests_ValidateEndpoints() [Fact] public void FailOverTests_GetNoDynamicClient() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://azure.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); From e0a910bdad661289d6d87cd04fe1950eac81e2a7 Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:49:27 -0700 Subject: [PATCH 02/22] add IsAzureAppConfigurationSource extension (#608) * add IsAzureAppConfigurationSource * nit: format --------- Co-authored-by: Sami Sadfa --- .../ConfigurationSourceExtensions.cs | 22 +++++++++++++++++++ .../RequestTracingOptions.cs | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs new file mode 100644 index 00000000..af7f1f03 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + /// + /// Provides extension methods for configuration sources. + /// + public static class ConfigurationSourceExtensions + { + /// + /// Determines whether the specified configuration source is an Azure App Configuration source. + /// + /// The configuration source to check. + /// true if the specified source is an Azure App Configuration source; otherwise, false. + public static bool IsAzureAppConfigurationSource(this IConfigurationSource source) + { + return source is AzureAppConfigurationSource; + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs index 7b06535b..3838ee95 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs @@ -66,7 +66,7 @@ internal class RequestTracingOptions /// /// Checks whether any tracing feature is used. /// - /// True if any tracing feature is used, otherwise false. + /// true if any tracing feature is used, otherwise false. public bool UsesAnyTracingFeature() { return IsLoadBalancingEnabled || IsSignalRUsed; From 752888889394f2c8db68071c9177fc63ed837d24 Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:02:32 -0700 Subject: [PATCH 03/22] update a comment to reflect that `ArgumentException` is not suppressed when the `optional` parameter is set to true (#599) * update comment * address comment --------- Co-authored-by: Sami Sadfa --- .../AzureAppConfigurationExtensions.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs index dc022bac..8b4bf8c8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs @@ -33,7 +33,9 @@ private static bool IsProviderDisabled() /// /// The configuration builder to add key-values to. /// The connection string used to connect to the configuration store. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, @@ -48,7 +50,9 @@ public static IConfigurationBuilder AddAzureAppConfiguration( /// /// The configuration builder to add key-values to. /// The list of connection strings used to connect to the configuration store and its replicas. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, @@ -63,7 +67,9 @@ public static IConfigurationBuilder AddAzureAppConfiguration( /// /// The configuration builder to add key-values to. /// A callback used to configure Azure App Configuration options. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, From f7ec121d7cf2e569f7934cbcb81c5cabbd4cd897 Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:22:42 -0700 Subject: [PATCH 04/22] check etag on 200 for sentinel key (#607) Co-authored-by: Sami Sadfa --- .../Extensions/ConfigurationClientExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index d479ad6b..4032b365 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -31,7 +31,8 @@ public static async Task GetKeyValueChange(this ConfigurationCli try { Response response = await client.GetConfigurationSettingAsync(setting, onlyIfChanged: true, cancellationToken).ConfigureAwait(false); - if (response.GetRawResponse().Status == (int)HttpStatusCode.OK) + if (response.GetRawResponse().Status == (int)HttpStatusCode.OK && + !response.Value.ETag.Equals(setting.ETag)) { return new KeyValueChange { From 6fc0f9ad734bbbcf6e735298275e78c46b88c855 Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:45:13 -0800 Subject: [PATCH 05/22] Fix regressed unit tests (#610) * fix first batch * tests fixed --- .../FeatureManagementTests.cs | 4 +-- .../KeyVaultReferenceTests.cs | 4 +-- .../LoggingTests.cs | 12 +++---- tests/Tests.AzureAppConfiguration/MapTests.cs | 14 ++++---- .../PushRefreshTests.cs | 2 +- .../RefreshTests.cs | 34 ++++++++----------- .../Tests.AzureAppConfiguration/TestHelper.cs | 5 +++ 7 files changed, 38 insertions(+), 37 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 44c20b4d..a05fee93 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -1724,7 +1724,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .Build(); Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); - FirstKeyValue.Value = "newValue1"; + FirstKeyValue = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -1817,7 +1817,7 @@ public async Task MapTransformFeatureFlagWithRefresh() Assert.Equal("TestValue1", config["TestKey1"]); Assert.Equal("NoUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - FirstKeyValue.Value = "newValue1"; + FirstKeyValue = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", value: @" diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index 06c88040..ea520efc 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -743,7 +743,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal(_secretValue, config[_kv.Key]); // Update sentinel key-value - sentinelKv.Value = "Value2"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "Value2"); Thread.Sleep(refreshInterval); await refresher.RefreshAsync(); @@ -815,7 +815,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal(_secretValue, config[_kv.Key]); // Update sentinel key-value to trigger refresh operation - sentinelKv.Value = "Value2"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "Value2"); Thread.Sleep(refreshInterval); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/LoggingTests.cs b/tests/Tests.AzureAppConfiguration/LoggingTests.cs index 547c65bd..6fcec29f 100644 --- a/tests/Tests.AzureAppConfiguration/LoggingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoggingTests.cs @@ -239,7 +239,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("SentinelValue", config["SentinelKey"]); // Update sentinel key-value to trigger refreshAll operation - sentinelKv.Value = "UpdatedSentinelValue"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "UpdatedSentinelValue"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -377,7 +377,7 @@ public async Task ValidateFailoverToDifferentEndpointMessageLoggedAfterFailover( .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -389,7 +389,7 @@ public async Task ValidateFailoverToDifferentEndpointMessageLoggedAfterFailover( .Throws(new RequestFailedException(HttpStatusCodes.TooManyRequests, "Too many requests")); mockClient2.Setup(c => c.ToString()).Returns("client"); - FirstKeyValue.Value = "TestValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "TestValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -431,7 +431,7 @@ public async Task ValidateConfigurationUpdatedSuccessLoggedDuringRefresh() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -480,7 +480,7 @@ public async Task ValidateCorrectEndpointLoggedOnConfigurationUpdate() }) .Build(); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -528,7 +528,7 @@ public async Task ValidateCorrectKeyValueLoggedDuringRefresh() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/MapTests.cs b/tests/Tests.AzureAppConfiguration/MapTests.cs index 623ae477..cdf15f4e 100644 --- a/tests/Tests.AzureAppConfiguration/MapTests.cs +++ b/tests/Tests.AzureAppConfiguration/MapTests.cs @@ -181,7 +181,7 @@ public async Task MapTransformWithRefresh() Assert.Equal("TestValue1 mapped first", config["TestKey1"]); Assert.Equal("TestValue2 second", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -232,8 +232,8 @@ public async Task MapTransformSettingKeyWithRefresh() Assert.Null(config["TestKey1"]); Assert.Equal("TestValue2", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -282,8 +282,8 @@ public async Task MapTransformSettingLabelWithRefresh() Assert.Equal("TestValue1 changed", config["TestKey1"]); Assert.Equal("TestValue2 changed", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -523,8 +523,8 @@ public async Task MapTransformSettingKeyWithLogAndRefresh() Assert.Null(config["TestKey1"]); Assert.Equal("TestValue2", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs b/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs index c4c7c38c..41e265d2 100644 --- a/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs @@ -347,7 +347,7 @@ public async Task RefreshAsyncUpdatesConfig() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); refresher.ProcessPushNotification(_pushNotificationList.First(), TimeSpan.FromSeconds(0)); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 6edc1a9a..dc904edc 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -208,7 +208,7 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue"); // Wait for the cache to expire Thread.Sleep(1500); @@ -221,7 +221,6 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() [Fact] public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() { - var keyValueCollection = new List(_kvCollection); IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -244,7 +243,7 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -259,7 +258,6 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() [Fact] public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() { - var keyValueCollection = new List(_kvCollection); IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -282,7 +280,7 @@ public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -353,7 +351,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.First().Value = "newValue"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue"); keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire @@ -426,8 +424,8 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ElementAt(0).Value = "newValue1"; - keyValueCollection.ElementAt(1).Value = "newValue2"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue1"); + keyValueCollection[1] = TestHelpers.ChangeValue(keyValueCollection[1], "newValue2"); keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire @@ -499,7 +497,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue1", config["TestKey1"]); Assert.Equal(1, requestCount); - keyValueCollection.First().Value = "newValue"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue"); // Simulate simultaneous refresh calls with expired cache from multiple threads var task1 = Task.Run(() => WaitAndRefresh(refresher, 1500)); @@ -606,7 +604,7 @@ public async Task RefreshTests_TryRefreshAsyncUpdatesConfigurationAndReturnsTrue .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire Thread.Sleep(1500); @@ -702,7 +700,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire Thread.Sleep(1500); @@ -823,7 +821,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + keyValueCollection = keyValueCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -874,7 +872,7 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel2", config["TestKeyWithMultipleLabels"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -890,8 +888,7 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire [Fact] public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfig() { - var keyValueCollection = new List(_kvCollection); - ConfigurationSetting refreshRegisteredSetting = keyValueCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); + ConfigurationSetting refreshRegisteredSetting = _kvCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); var mockClient = GetMockConfigurationClient(); IConfigurationRefresher refresher = null; @@ -916,7 +913,7 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel2", config["TestKeyWithMultipleLabels"]); - refreshRegisteredSetting.Value = "UpdatedValueForLabel1"; + _kvCollection[_kvCollection.IndexOf(refreshRegisteredSetting)] = TestHelpers.ChangeValue(refreshRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire Thread.Sleep(1500); @@ -933,8 +930,7 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi [Fact] public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() { - var keyValueCollection = new List(_kvCollection); - ConfigurationSetting refreshAllRegisteredSetting = keyValueCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); + ConfigurationSetting refreshAllRegisteredSetting = _kvCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); var mockClient = GetMockConfigurationClient(); IConfigurationRefresher refresher = null; @@ -959,7 +955,7 @@ public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel1", config["TestKeyWithMultipleLabels"]); - refreshAllRegisteredSetting.Value = "UpdatedValueForLabel1"; + _kvCollection[_kvCollection.IndexOf(refreshAllRegisteredSetting)] = TestHelpers.ChangeValue(refreshAllRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire Thread.Sleep(1500); diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index bc7989b2..75a23afe 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -115,6 +115,11 @@ public static ConfigurationSetting CloneSetting(ConfigurationSetting setting) return ConfigurationModelFactory.ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ContentType, setting.ETag, setting.LastModified); } + public static ConfigurationSetting ChangeValue(ConfigurationSetting setting, string value) + { + return ConfigurationModelFactory.ConfigurationSetting(setting.Key, value, setting.Label, setting.ContentType, new ETag(Guid.NewGuid().ToString()), setting.LastModified); + } + public static List LoadJsonSettingsFromFile(string path) { List _kvCollection = new List(); From 90835cf5efbfb26d6de6068e4c31c3050947b78a Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:22:00 -0800 Subject: [PATCH 06/22] ensure failed exit codes are returned to ci.yml (#609) --- test.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test.ps1 b/test.ps1 index 19c80dff..9acc5054 100644 --- a/test.ps1 +++ b/test.ps1 @@ -3,7 +3,17 @@ $ErrorActionPreference = "Stop" $dotnet = & "$PSScriptRoot/build/resolve-dotnet.ps1" & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration\Tests.AzureAppConfiguration.csproj" --logger trx + +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration.AspNetCore\Tests.AzureAppConfiguration.AspNetCore.csproj" --logger trx + +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration.Functions.Worker\Tests.AzureAppConfiguration.Functions.Worker.csproj" --logger trx exit $LASTEXITCODE \ No newline at end of file From a544fd008aa8367c6d34a0e375fe196d0502a514 Mon Sep 17 00:00:00 2001 From: Zhiyuan Liang <141655842+zhiyuanliang-ms@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:33:53 +0800 Subject: [PATCH 07/22] Remove variant config ref (#614) * remove variant config ref * update testcase --- .../Constants/RequestTracingConstants.cs | 1 - .../FeatureManagement/FeatureFlagTracing.cs | 14 +---------- .../FeatureManagementConstants.cs | 1 - .../FeatureManagementKeyValueAdapter.cs | 25 ------------------- .../FeatureManagement/FeatureVariant.cs | 2 -- .../FeatureManagementTests.cs | 7 ++---- 6 files changed, 3 insertions(+), 47 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs index 15e862b6..1084c274 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs @@ -43,7 +43,6 @@ internal class RequestTracingConstants public const string FeatureFlagUsesTelemetryTag = "Telemetry"; public const string FeatureFlagUsesSeedTag = "Seed"; public const string FeatureFlagMaxVariantsKey = "MaxVariants"; - public const string FeatureFlagUsesVariantConfigurationReferenceTag = "ConfigRef"; public const string DiagnosticHeaderActivityName = "Azure.CustomDiagnosticHeaders"; public const string CorrelationContextHeader = "Correlation-Context"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs index f48b5220..8c696e49 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs @@ -25,7 +25,6 @@ internal class FeatureFlagTracing public bool UsesTargetingFilter { get; set; } = false; public bool UsesSeed { get; set; } = false; public bool UsesTelemetry { get; set; } = false; - public bool UsesVariantConfigurationReference { get; set; } = false; public int MaxVariants { get; set; } public bool UsesAnyFeatureFilter() @@ -35,7 +34,7 @@ public bool UsesAnyFeatureFilter() public bool UsesAnyTracingFeature() { - return UsesSeed || UsesTelemetry || UsesVariantConfigurationReference; + return UsesSeed || UsesTelemetry; } public void ResetFeatureFlagTracing() @@ -46,7 +45,6 @@ public void ResetFeatureFlagTracing() UsesTargetingFilter = false; UsesSeed = false; UsesTelemetry = false; - UsesVariantConfigurationReference = false; MaxVariants = 0; } @@ -147,16 +145,6 @@ public string CreateFeaturesString() sb.Append(RequestTracingConstants.FeatureFlagUsesSeedTag); } - if (UsesVariantConfigurationReference) - { - if (sb.Length > 0) - { - sb.Append(RequestTracingConstants.Delimiter); - } - - sb.Append(RequestTracingConstants.FeatureFlagUsesVariantConfigurationReferenceTag); - } - if (UsesTelemetry) { if (sb.Length > 0) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index c6d86d84..6d385797 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -30,7 +30,6 @@ internal class FeatureManagementConstants public const string Parameters = "parameters"; public const string Variant = "variant"; public const string ConfigurationValue = "configuration_value"; - public const string ConfigurationReference = "configuration_reference"; public const string StatusOverride = "status_override"; public const string DefaultWhenDisabled = "default_when_disabled"; public const string DefaultWhenEnabled = "default_when_enabled"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 2b0e0833..b6d137f3 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -187,13 +187,6 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea (string.IsNullOrEmpty(kvp.Key) ? "" : $":{kvp.Key}"), kvp.Value)); } - if (featureVariant.ConfigurationReference != null) - { - _featureFlagTracing.UsesVariantConfigurationReference = true; - - keyValues.Add(new KeyValuePair($"{variantsPath}:{FeatureManagementConstants.ConfigurationReference}", featureVariant.ConfigurationReference)); - } - if (featureVariant.StatusOverride != null) { keyValues.Add(new KeyValuePair($"{variantsPath}:{FeatureManagementConstants.StatusOverride}", featureVariant.StatusOverride)); @@ -1157,24 +1150,6 @@ private FeatureVariant ParseFeatureVariant(ref Utf8JsonReader reader, string set break; } - case FeatureManagementConstants.ConfigurationReference: - { - if (reader.Read() && reader.TokenType == JsonTokenType.String) - { - featureVariant.ConfigurationReference = reader.GetString(); - } - else if (reader.TokenType != JsonTokenType.Null) - { - throw CreateFeatureFlagFormatException( - FeatureManagementConstants.ConfigurationReference, - settingKey, - reader.TokenType.ToString(), - JsonTokenType.String.ToString()); - } - - break; - } - case FeatureManagementConstants.ConfigurationValue: { if (reader.Read()) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs index 87c5c0b1..7bfd9d6f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs @@ -11,8 +11,6 @@ internal class FeatureVariant public JsonElement ConfigurationValue { get; set; } - public string ConfigurationReference { get; set; } - public string StatusOverride { get; set; } } } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index a05fee93..96d2cf08 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -435,7 +435,6 @@ public class FeatureManagementTests }, { ""name"": ""Small"", - ""configuration_reference"": ""ShoppingCart:Small"", ""status_override"": ""Disabled"" } ], @@ -1868,7 +1867,6 @@ public void WithVariants() Assert.Equal("Big", config["feature_management:feature_flags:0:variants:0:name"]); Assert.Equal("600px", config["feature_management:feature_flags:0:variants:0:configuration_value"]); Assert.Equal("Small", config["feature_management:feature_flags:0:variants:1:name"]); - Assert.Equal("ShoppingCart:Small", config["feature_management:feature_flags:0:variants:1:configuration_reference"]); Assert.Equal("Disabled", config["feature_management:feature_flags:0:variants:1:status_override"]); Assert.Equal("Small", config["feature_management:feature_flags:0:allocation:default_when_disabled"]); Assert.Equal("Small", config["feature_management:feature_flags:0:allocation:default_when_enabled"]); @@ -2023,7 +2021,7 @@ public void ThrowsOnIncorrectJsonTypes() var settings = new List() { CreateFeatureFlag("Feature1", variantsJsonString: @"[{""name"": 1}]"), - CreateFeatureFlag("Feature2", variantsJsonString: @"[{""configuration_reference"": true}]"), + CreateFeatureFlag("Feature2", requirementType: "2"), CreateFeatureFlag("Feature3", variantsJsonString: @"[{""status_override"": []}]"), CreateFeatureFlag("Feature4", seed: "{}"), CreateFeatureFlag("Feature5", defaultWhenDisabled: "5"), @@ -2038,8 +2036,7 @@ public void ThrowsOnIncorrectJsonTypes() CreateFeatureFlag("Feature14", telemetryEnabled: "14"), CreateFeatureFlag("Feature15", telemetryMetadataJsonString: @"{""key"": 15}"), CreateFeatureFlag("Feature16", clientFiltersJsonString: @"[{""name"": 16}]"), - CreateFeatureFlag("Feature17", clientFiltersJsonString: @"{""key"": [{""name"": ""name"", ""parameters"": 17}]}"), - CreateFeatureFlag("Feature18", requirementType: "18") + CreateFeatureFlag("Feature17", clientFiltersJsonString: @"{""key"": [{""name"": ""name"", ""parameters"": 17}]}") }; var mockResponse = new Mock(); From 40768ff7284f87f553f4cedcf0c61e03a40fcc86 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:41:17 -0800 Subject: [PATCH 08/22] Include discovered replicas in replica count for request tracing (#613) * count total clients from client manager for replica count * set replica count whenevr calling getclients --- .../AzureAppConfigurationProvider.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b5dd42f1..89ba0510 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -208,6 +208,11 @@ public async Task RefreshAsync(CancellationToken cancellationToken) IEnumerable clients = _configClientManager.GetClients(); + if (_requestTracingOptions != null) + { + _requestTracingOptions.ReplicaCount = clients.Count() - 1; + } + // // Filter clients based on their backoff status clients = clients.Where(client => @@ -612,6 +617,11 @@ private async Task LoadAsync(bool ignoreFailures, CancellationToken cancellation { IEnumerable clients = _configClientManager.GetClients(); + if (_requestTracingOptions != null) + { + _requestTracingOptions.ReplicaCount = clients.Count() - 1; + } + if (await TryInitializeAsync(clients, startupExceptions, cancellationToken).ConfigureAwait(false)) { break; @@ -966,7 +976,6 @@ private void SetRequestTracingOptions() IsDevEnvironment = TracingUtils.IsDevEnvironment(), IsKeyVaultConfigured = _options.IsKeyVaultConfigured, IsKeyVaultRefreshConfigured = _options.IsKeyVaultRefreshConfigured, - ReplicaCount = _options.Endpoints?.Count() - 1 ?? _options.ConnectionStrings?.Count() - 1 ?? 0, FeatureFlagTracing = _options.FeatureFlagTracing, IsLoadBalancingEnabled = _options.LoadBalancingEnabled }; From 5e6a012e18959f4eed4a82057a7561b081f1d33e Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:51:56 -0800 Subject: [PATCH 09/22] Add `RegisterAll` API to enable monitoring collections of key-values for refresh (#574) * WIP * WIP testing out client extensions methods * WIP added selectors to multikeywatchers * remove unused property * WIP check for registerall changes to change refreshall * WIP * WIP fixing types and reslving errors * WIP fixing client extensions class * WIP * WIP update feature flag logic * WIP client extensions * WIP reload all flags on change * WIP * WIP fixing tests to return response for getconfigurationsettingsasync * WIP etag for tests * fix watchedcollections null * WIP tests, working for examples * remove unused variables * update to newest sdk version, remove unused * WIP fixing tests * WIP reworking testing to work with new etag approach * tests passing, fix mockasyncpageable * update sdk package version * fix loghelper, tests * WIP fixing aspages tests * revert watchesfeatureflags test * update test again * WIP * fixing watchconditions * separate selected key value collections from feature flag collections, separate selectors, add new methods to support new logic * comment and naming updates * fixing unit tests, namespace of defining/calling code needs to be same * fixing tests using AsPages * fix tests with pageablemanager * format * fix tests * fix tests * remove unused extension test class * fix comment, capitalization * check etag on 200, fix tests * add registerall test, fix refresh tests * fix condition for pages and old match conditions * WIP fixing PR comments, tests * check status after advancing existing etag enumerator * move around refresh logic * null check page etag, revert break to existing keys check in getrefreshedcollections * fix loadselected, replace selectedkvwatchers with registerall refresh time * fix comment in options * clean up tests * PR comments * PR comments * don't allow both registerall and register * fix check for calls to both register methods * PR comments for rename/small changes * fix compile error * simplify refreshasync path, fix naming from comments * remove redundant if check * simplify logic for minrefreshinterval * fix smaller comments * call loadselected when refreshing collection, separate data for individual refresh * in progress change to registerall include ff * fix load order * fix comments, rename logging constants to match new behavior * pr comments, refactor refreshasync * clean up etags dictionary creation * PR comments * add uncommitted changes to testhelper * update tests for registerall with feature flags, check ff keys to remove flags on refresh * PR comments * PR comments * use invalidoperationexception in configurerefresh, update loggingconstants to match behavior * remove unused changes --- .../AzureAppConfigurationOptions.cs | 113 ++- .../AzureAppConfigurationProvider.cs | 654 +++++++++++------- .../AzureAppConfigurationRefreshOptions.cs | 12 + .../ConfigurationSettingPageExtensions.cs | 33 + .../Constants/LoggingConstants.cs | 5 +- .../ConfigurationClientExtensions.cs | 129 +--- .../Extensions/StringExtensions.cs | 9 +- .../FeatureManagement/FeatureFlagOptions.cs | 3 +- .../GetKeyValueChangeCollectionOptions.cs | 13 - .../IConfigurationSettingPageIterator.cs | 13 + .../LogHelper.cs | 15 +- .../Models/KeyValueSelector.cs | 5 + .../Azure.Core.Testing/MockResponse.cs | 2 + .../FeatureManagementTests.cs | 100 ++- .../KeyVaultReferenceTests.cs | 4 +- .../RefreshTests.cs | 165 +++++ .../Tests.AzureAppConfiguration/TestHelper.cs | 60 +- 17 files changed, 865 insertions(+), 470 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7d9a9cad..9391f21b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -23,12 +23,13 @@ public class AzureAppConfigurationOptions private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); - private List _changeWatchers = new List(); - private List _multiKeyWatchers = new List(); + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors = new List(); + private List _selectors; private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; // The following set is sorted in descending order. // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. @@ -62,19 +63,29 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// A collection of . + /// A collection of specified by user. /// - internal IEnumerable KeyValueSelectors => _kvSelectors; + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } /// /// A collection of . /// - internal IEnumerable ChangeWatchers => _changeWatchers; + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable FeatureFlagWatchers => _ffWatchers; /// /// A collection of . @@ -96,11 +107,15 @@ internal IEnumerable Adapters internal IEnumerable KeyPrefixes => _keyPrefixes; /// - /// An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. /// - /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. /// @@ -142,6 +157,9 @@ public AzureAppConfigurationOptions() new JsonKeyValueAdapter(), new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } /// @@ -170,22 +188,30 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter throw new ArgumentNullException(nameof(keyFilter)); } + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + if (string.IsNullOrWhiteSpace(labelFilter)) { labelFilter = LabelFilter.Null; } - // Do not support * and , for label filter for now. - if (labelFilter.Contains('*') || labelFilter.Contains(',')) + if (!_selectCalled) { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + _selectors.Clear(); + + _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { KeyFilter = keyFilter, LabelFilter = labelFilter }); + return this; } @@ -201,7 +227,14 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) throw new ArgumentNullException(nameof(name)); } - _kvSelectors.AppendUnique(new KeyValueSelector + if (!_selectCalled) + { + _selectors.Clear(); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector { SnapshotName = name }); @@ -212,7 +245,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh on an individual flag level. + /// All loaded feature flags will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) @@ -237,25 +270,22 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = options.Label == null ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true }); } - foreach (var featureFlagSelector in options.FeatureFlagSelectors) + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - var featureFlagFilter = featureFlagSelector.KeyFilter; - var labelFilter = featureFlagSelector.LabelFilter; + _selectors.AppendUnique(featureFlagSelector); - Select(featureFlagFilter, labelFilter); - - _multiKeyWatchers.AppendUnique(new KeyValueWatcher + _ffWatchers.AppendUnique(new KeyValueWatcher { - Key = featureFlagFilter, - Label = labelFilter, + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); - } return this; @@ -376,18 +406,41 @@ public AzureAppConfigurationOptions ConfigureClientOptions(ActionA callback used to configure Azure App Configuration refresh options. public AzureAppConfigurationOptions ConfigureRefresh(Action configure) { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + var refreshOptions = new AzureAppConfigurationRefreshOptions(); configure?.Invoke(refreshOptions); - if (!refreshOptions.RefreshRegistrations.Any()) + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); + } + + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) { - throw new ArgumentException($"{nameof(ConfigureRefresh)}() must have at least one key-value registered for refresh."); + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); } - foreach (var item in refreshOptions.RefreshRegistrations) + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else { - item.RefreshInterval = refreshOptions.RefreshInterval; - _changeWatchers.Add(item); + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } } return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b5dd42f1..ebe8992c 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -4,7 +4,6 @@ using Azure; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using Microsoft.Extensions.Logging; using System; @@ -32,9 +31,13 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); + private Dictionary _watchedIndividualKvs = new Dictionary(); + private HashSet _ffKeys = new HashSet(); + private Dictionary> _kvEtags = new Dictionary>(); + private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); + private DateTimeOffset _nextCollectionRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -108,11 +111,18 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); + IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (watchers.Any()) + bool hasWatchers = watchers.Any(); + TimeSpan minWatcherRefreshInterval = hasWatchers ? watchers.Min(w => w.RefreshInterval) : TimeSpan.MaxValue; + + if (options.RegisterAllEnabled) + { + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); + } + else if (hasWatchers) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + MinRefreshInterval = minWatcherRefreshInterval; } else { @@ -194,13 +204,15 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableMultiKeyWatchers = _options.MultiKeyWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); + bool isRefreshDue = utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableWatchers.Any() && - !refreshableMultiKeyWatchers.Any() && + !refreshableIndividualKvWatchers.Any() && + !refreshableFfWatchers.Any() && + !isRefreshDue && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -249,179 +261,166 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary watchedSettings = null; + Dictionary> kvEtags = null; + Dictionary> ffEtags = null; + HashSet ffKeys = null; + Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - List changedKeyValuesCollection = null; Dictionary data = null; + Dictionary ffCollectionData = null; + bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => - { - data = null; - watchedSettings = null; - keyValueChanges = new List(); - changedKeyValuesCollection = null; - refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); - logDebugBuilder.Clear(); - logInfoBuilder.Clear(); - - foreach (KeyValueWatcher changeWatcher in refreshableWatchers) - { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; - - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); - - KeyValueChange change = default; - - // - // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) - { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - else - { - // Load the key-value in case the previous load attempts had failed - - try - { - await CallWithRequestTracing( - async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) - { - watchedKv = null; - } - - if (watchedKv != null) - { - change = new KeyValueChange() - { - Key = watchedKv.Key, - Label = watchedKv.Label.NormalizeNull(), - Current = watchedKv, - ChangeType = KeyValueChangeType.Modified - }; - } - } - - // Check if a change has been detected in the key-value registered for refresh - if (change.ChangeType != KeyValueChangeType.None) - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); - keyValueChanges.Add(change); - - if (changeWatcher.RefreshAll) - { - refreshAll = true; - break; - } - } - else - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - } - } + { + kvEtags = null; + ffEtags = null; + ffKeys = null; + watchedIndividualKvs = null; + keyValueChanges = new List(); + data = null; + ffCollectionData = null; + ffCollectionUpdated = false; + refreshAll = false; + logDebugBuilder.Clear(); + logInfoBuilder.Clear(); + Uri endpoint = _configClientManager.GetEndpointForClient(client); - if (refreshAll) + if (_options.RegisterAllEnabled) + { + // Get key value collection changes if RegisterAll was called + if (isRefreshDue) { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; + refreshAll = await HaveCollectionsChanged( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector), + _kvEtags, + client, + cancellationToken).ConfigureAwait(false); } + } + else + { + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); + } - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + if (refreshAll) + { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true, + // or if any key-value collection change was detected. + kvEtags = new Dictionary>(); + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, ffKeys, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } - if (!changedKeyValuesCollection.Any()) + // Get feature flag changes + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - }, - cancellationToken) - .ConfigureAwait(false); + KeyFilter = watcher.Key, + LabelFilter = watcher.Label, + IsFeatureFlagSelector = true + }), + _ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) + { + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + ffKeys, + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + }, + cancellationToken) + .ConfigureAwait(false); - if (!refreshAll) + if (refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) + // Invalidate all the cached KeyVault secrets + foreach (IKeyValueAdapter adapter in _options.Adapters) { - UpdateNextRefreshTime(changeWatcher); + adapter.OnChangeDetected(); } - foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); - if (change.ChangeType == KeyValueChangeType.Modified) - { - ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + UpdateNextRefreshTime(changeWatcher); + } + } + else + { + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - foreach (Func> func in _options.Mappers) - { - setting = await func(setting).ConfigureAwait(false); - } + await ProcessKeyValueChangesAsync(keyValueChanges, _mappedData, watchedIndividualKvs).ConfigureAwait(false); - if (setting == null) - { - _mappedData.Remove(change.Key); - } - else - { - _mappedData[change.Key] = setting; - } - } - else if (change.ChangeType == KeyValueChangeType.Deleted) + if (ffCollectionUpdated) + { + // Remove all feature flag keys that are not present in the latest loading of feature flags, but were loaded previously + foreach (string key in _ffKeys.Except(ffKeys)) { - _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + _mappedData.Remove(key); } - // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting - foreach (IKeyValueAdapter adapter in _options.Adapters) + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + foreach (KeyValuePair kvp in mappedFfData) { - // If the current setting is null, try to pass the previous setting instead - if (change.Current != null) - { - adapter.OnChangeDetected(change.Current); - } - else if (change.Previous != null) - { - adapter.OnChangeDetected(change.Previous); - } + _mappedData[kvp.Key] = kvp.Value; } } - } - else - { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - - // Invalidate all the cached KeyVault secrets - foreach (IKeyValueAdapter adapter in _options.Adapters) - { - adapter.OnChangeDetected(); - } - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + // + // update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { UpdateNextRefreshTime(changeWatcher); } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || changedKeyValuesCollection?.Any() == true || keyValueChanges.Any()) + if (isRefreshDue) { - _watchedSettings = watchedSettings; + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) + { + _watchedIndividualKvs = watchedIndividualKvs ?? _watchedIndividualKvs; + + _ffEtags = ffEtags ?? _ffEtags; + + _kvEtags = kvEtags ?? _kvEtags; + + _ffKeys = ffKeys ?? _ffKeys; if (logDebugBuilder.Length > 0) { @@ -432,6 +431,7 @@ await CallWithRequestTracing( { _logger.LogInformation(logInfoBuilder.ToString().Trim()); } + // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. @@ -555,14 +555,21 @@ private void SetDirty(TimeSpan? maxDelay) { DateTimeOffset nextRefreshTime = AddRandomDelay(DateTimeOffset.UtcNow, maxDelay ?? DefaultMaxSetDirtyDelay); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + if (_options.RegisterAllEnabled) { - changeWatcher.NextRefreshTime = nextRefreshTime; + _nextCollectionRefreshTime = nextRefreshTime; + } + else + { + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) + { + kvWatcher.NextRefreshTime = nextRefreshTime; + } } - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { - changeWatcher.NextRefreshTime = nextRefreshTime; + featureFlagWatcher.NextRefreshTime = nextRefreshTime; } } @@ -707,34 +714,44 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary watchedSettings = null; + Dictionary> kvEtags = new Dictionary>(); + Dictionary> ffEtags = new Dictionary>(); + Dictionary watchedIndividualKvs = null; + HashSet ffKeys = new HashSet(); await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - data = await LoadSelectedKeyValues( + data = await LoadSelected( client, + kvEtags, + ffEtags, + _options.Selectors, + ffKeys, cancellationToken) .ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh( + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) .ConfigureAwait(false); - - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); }, cancellationToken) .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + if (_options.RegisterAllEnabled) + { + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + if (data != null) { // Invalidate all the cached KeyVault secrets @@ -744,51 +761,71 @@ await ExecuteWithFailOverPolicyAsync( } Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; + _mappedData = mappedData; + _kvEtags = kvEtags; + _ffEtags = ffEtags; + _watchedIndividualKvs = watchedIndividualKvs; + _ffKeys = ffKeys; } } - private async Task> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task> LoadSelected( + ConfigurationClient client, + Dictionary> kvEtags, + Dictionary> ffEtags, + IEnumerable selectors, + HashSet ffKeys, + CancellationToken cancellationToken) { - var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary data = new Dictionary(); - // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || - !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); - - if (useDefaultQuery) + foreach (KeyValueSelector loadOption in selectors) { - // Load all key-values with the null label. - var selector = new SettingSelector - { - KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null - }; - - await CallWithRequestTracing(async () => + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + var selector = new SettingSelector() { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); - } + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }; - foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) - { - IAsyncEnumerable settingsEnumerable; + var matchConditions = new List(); - if (string.IsNullOrEmpty(loadOption.SnapshotName)) - { - settingsEnumerable = client.GetConfigurationSettingsAsync( - new SettingSelector + await CallWithRequestTracing(async () => + { + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages(_options.ConfigurationSettingPageIterator).ConfigureAwait(false)) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }, - cancellationToken); + using Response response = page.GetRawResponse(); + + ETag serverEtag = (ETag)response.Headers.ETag; + + foreach (ConfigurationSetting setting in page.Values) + { + data[setting.Key] = setting; + + if (loadOption.IsFeatureFlagSelector) + { + ffKeys.Add(setting.Key); + } + } + + matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); + } + }).ConfigureAwait(false); + + if (loadOption.IsFeatureFlagSelector) + { + ffEtags[loadOption] = matchConditions; + } + else + { + kvEtags[loadOption] = matchConditions; + } } else { @@ -808,38 +845,42 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + IAsyncEnumerable settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( loadOption.SnapshotName, cancellationToken); - } - await CallWithRequestTracing(async () => - { - await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + await CallWithRequestTracing(async () => { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + { + data[setting.Key] = setting; + } + }).ConfigureAwait(false); + } } - return serverData; + return data; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh( + ConfigurationClient client, + IDictionary existingSettings, + CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + var watchedIndividualKvs = new Dictionary(); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -857,61 +898,84 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSettings; + return watchedIndividualKvs; } - private Dictionary UpdateWatchedKeyValueCollections(Dictionary watchedSettings, IDictionary existingSettings) + private async Task RefreshIndividualKvWatchers( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableIndividualKvWatchers, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher kvWatcher in refreshableIndividualKvWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; - foreach (ConfigurationSetting setting in currentKeyValues) + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + KeyValueChange change = default; + + // + // Find if there is a change associated with watcher + if (_watchedIndividualKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - } + else + { + // Load the key-value in case the previous load attempts had failed - return watchedSettings; - } + try + { + await CallWithRequestTracing( + async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) + { + watchedKv = null; + } - private async Task> GetRefreshedKeyValueCollections( - IEnumerable multiKeyWatchers, - ConfigurationClient client, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) - { - var keyValueChanges = new List(); + if (watchedKv != null) + { + change = new KeyValueChange() + { + Key = watchedKv.Key, + Label = watchedKv.Label.NormalizeNull(), + Current = watchedKv, + ChangeType = KeyValueChangeType.Modified + }; + } + } - foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) - { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, _watchedSettings.Values); + // Check if a change has been detected in the key-value registered for refresh + if (change.ChangeType != KeyValueChangeType.None) + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); + keyValueChanges.Add(change); - keyValueChanges.AddRange( - await client.GetKeyValueChangeCollection( - currentKeyValues, - new GetKeyValueChangeCollectionOptions - { - KeyFilter = changeWatcher.Key, - Label = changeWatcher.Label.NormalizeNull(), - RequestTracingEnabled = _requestTracingEnabled, - RequestTracingOptions = _requestTracingOptions - }, - logDebugBuilder, - logInfoBuilder, - endpoint, - cancellationToken) - .ConfigureAwait(false)); + if (kvWatcher.RefreshAll) + { + return true; + } + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + } } - return keyValueChanges; + return false; } private void SetData(IDictionary data) @@ -1179,30 +1243,6 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingSettings) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.StartsWith(keyPrefix) && kv.Label == label.NormalizeNull(); - }); - } - else - { - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.Equals(key) && kv.Label == label.NormalizeNull(); - }); - } - - return currentKeyValues; - } - private void EnsureAssemblyInspected() { if (!_isAssemblyInspected) @@ -1248,6 +1288,88 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } + private async Task HaveCollectionsChanged( + IEnumerable selectors, + Dictionary> pageEtags, + ConfigurationClient client, + CancellationToken cancellationToken) + { + bool haveCollectionsChanged = false; + + foreach (KeyValueSelector selector in selectors) + { + if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => haveCollectionsChanged = await client.HaveCollectionsChanged( + selector, + matchConditions, + _options.ConfigurationSettingPageIterator, + cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (haveCollectionsChanged) + { + return true; + } + } + + return haveCollectionsChanged; + } + + private async Task ProcessKeyValueChangesAsync( + IEnumerable keyValueChanges, + Dictionary mappedData, + Dictionary watchedIndividualKvs) + { + foreach (KeyValueChange change in keyValueChanges) + { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + + if (change.ChangeType == KeyValueChangeType.Modified) + { + ConfigurationSetting setting = change.Current; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + watchedIndividualKvs[changeIdentifier] = settingCopy; + + foreach (Func> func in _options.Mappers) + { + setting = await func(setting).ConfigureAwait(false); + } + + if (setting == null) + { + mappedData.Remove(change.Key); + } + else + { + mappedData[change.Key] = setting; + } + } + else if (change.ChangeType == KeyValueChangeType.Deleted) + { + mappedData.Remove(change.Key); + + watchedIndividualKvs.Remove(changeIdentifier); + } + + // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting + foreach (IKeyValueAdapter adapter in _options.Adapters) + { + // If the current setting is null, try to pass the previous setting instead + if (change.Current != null) + { + adapter.OnChangeDetected(change.Current); + } + else if (change.Previous != null) + { + adapter.OnChangeDetected(change.Previous); + } + } + } + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index f3fb6c4a..cf2847a8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -14,6 +14,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); + internal bool RegisterAllEnabled { get; private set; } /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. @@ -50,6 +51,17 @@ public AzureAppConfigurationRefreshOptions Register(string key, string label = L return this; } + /// + /// Register all key-values loaded outside of to be refreshed when the configuration provider's triggers a refresh. + /// The instance can be obtained by calling . + /// + public AzureAppConfigurationRefreshOptions RegisterAll() + { + RegisterAllEnabled = true; + + return this; + } + /// /// Sets the cache expiration time for the key-values registered for refresh. Default value is 30 seconds. Must be greater than 1 second. /// Any refresh operation triggered using will not update the value for a key until the cached value for that key has expired. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs new file mode 100644 index 00000000..aba7684b --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs @@ -0,0 +1,33 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 86576a48..3bdcaecd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -20,14 +20,15 @@ internal class LoggingConstants // Successful update, debug log level public const string RefreshKeyValueRead = "Key-value read from App Configuration."; public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; - public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValueCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; - public const string RefreshFeatureFlagUpdated = "Feature flag updated."; + public const string RefreshFeatureFlagsUpdated = "Feature flags reloaded."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags reloaded."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index d479ad6b..67712231 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,12 +3,10 @@ // using Azure; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -65,131 +63,50 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetKeyValueChangeCollection( - this ConfigurationClient client, - IEnumerable keyValues, - GetKeyValueChangeCollectionOptions options, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IConfigurationSettingPageIterator pageIterator, CancellationToken cancellationToken) { - if (options == null) + if (matchConditions == null) { - throw new ArgumentNullException(nameof(options)); + throw new ArgumentNullException(nameof(matchConditions)); } - if (keyValues == null) + if (keyValueSelector == null) { - keyValues = Enumerable.Empty(); + throw new ArgumentNullException(nameof(keyValueSelector)); } - if (options.KeyFilter == null) + if (keyValueSelector.SnapshotName != null) { - options.KeyFilter = string.Empty; + throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); } - if (keyValues.Any(k => string.IsNullOrEmpty(k.Key))) + SettingSelector selector = new SettingSelector { - throw new ArgumentNullException($"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Key)}"); - } - - if (keyValues.Any(k => !string.Equals(k.Label.NormalizeNull(), options.Label.NormalizeNull()))) - { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Label)}"); - } - - if (keyValues.Any(k => k.Label != null && k.Label.Contains("*"))) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); - } - - var hasKeyValueCollectionChanged = false; - var selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label, - Fields = SettingFields.ETag | SettingFields.Key + KeyFilter = keyValueSelector.KeyFilter, + LabelFilter = keyValueSelector.LabelFilter }; - // Dictionary of eTags that we write to and use for comparison - var eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); + AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); - // Fetch e-tags for prefixed key-values that can be used to detect changes - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - hasKeyValueCollectionChanged = true; - break; - } + using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - // Check for any deletions - if (eTagMap.Any()) - { - hasKeyValueCollectionChanged = true; - } - - var changes = new List(); - - // If changes have been observed, refresh prefixed key-values - if (hasKeyValueCollectionChanged) + await foreach (Page page in pageable.AsPages(pageIterator, matchConditions).ConfigureAwait(false)) { - selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label - }; + using Response response = page.GetRawResponse(); - eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } + ETag serverEtag = (ETag)response.Headers.ETag; - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) + // Return true if the lists of etags are different + if ((!existingMatchConditionsEnumerator.MoveNext() || + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && + response.Status == (int)HttpStatusCode.OK) { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); + return true; } } - return changes; + // Need to check if pages were deleted and no change was found within the new shorter list of match conditions + return existingMatchConditionsEnumerator.MoveNext(); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 8b2c488d..61572030 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -5,18 +5,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { - internal static class LabelFilters - { - public static readonly string Null = "\0"; - - public static readonly string Any = "*"; - } - internal static class StringExtensions { public static string NormalizeNull(this string s) { - return s == LabelFilters.Null ? null : s; + return s == LabelFilter.Null ? null : s; } public static string ToBase64String(this string s) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs index 1e8beae6..26390762 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs @@ -102,7 +102,8 @@ public FeatureFlagOptions Select(string featureFlagFilter, string labelFilter = FeatureFlagSelectors.AppendUnique(new KeyValueSelector { KeyFilter = featureFlagPrefix, - LabelFilter = labelFilter + LabelFilter = labelFilter, + IsFeatureFlagSelector = true }); return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs deleted file mode 100644 index 5cb9b83d..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class GetKeyValueChangeCollectionOptions - { - public string KeyFilter { get; set; } - public string Label { get; set; } - public bool RequestTracingEnabled { get; set; } - public RequestTracingOptions RequestTracingOptions { get; set; } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs new file mode 100644 index 00000000..08c95751 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs @@ -0,0 +1,13 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IConfigurationSettingPageIterator + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 11442dcb..4f999406 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -25,14 +25,19 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) return $"{LoggingConstants.RefreshFeatureFlagsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagReadMessage(string key, string label, string endpoint) - { - return $"{LoggingConstants.RefreshFeatureFlagRead} Key:'{key}' Label:'{label}' Endpoint:'{endpoint?.TrimEnd('/')}'"; + public static string BuildFeatureFlagsUpdatedMessage() + { + return LoggingConstants.RefreshFeatureFlagsUpdated; + } + + public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) + { + return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagUpdatedMessage(string key) + public static string BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage() { - return $"{LoggingConstants.RefreshFeatureFlagUpdated} Key:'{key}'"; + return LoggingConstants.RefreshSelectedKeyValuesAndFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 5491d04d..54bda1a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -24,6 +24,11 @@ public class KeyValueSelector /// public string SnapshotName { get; set; } + /// + /// A boolean that signifies whether this selector is intended to select feature flags. + /// + public bool IsFeatureFlagSelector { get; set; } + /// /// Determines whether the specified object is equal to the current object. /// diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77..aaee0b9c 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -17,6 +17,8 @@ public MockResponse(int status, string reasonPhrase = null) { Status = status; ReasonPhrase = reasonPhrase; + + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); } public override int Status { get; } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 7e49e8ab..f7ebf127 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -768,19 +768,24 @@ public void UsesFeatureFlags() [Fact] public async Task WatchesFeatureFlags() { + var mockResponse = new MockResponse(200); + var featureFlags = new List { _kv }; - var mockResponse = new Mock(); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -820,7 +825,7 @@ public async Task WatchesFeatureFlags() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); featureFlags.Add(_kv2); @@ -839,11 +844,12 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -852,6 +858,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -910,17 +917,20 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -977,17 +987,20 @@ public async Task SkipRefreshIfCacheNotExpired() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -1096,17 +1109,22 @@ public void QueriesFeatureFlags() } [Fact] - public async Task UsesEtagForFeatureFlagRefresh() + public async Task DoesNotUseEtagForFeatureFlagRefresh() { + var mockAsyncPageable = new MockAsyncPageable(new List { _kv }); + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(new List { _kv })); + .Callback(() => mockAsyncPageable.UpdateCollection(new List { _kv })) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1137,6 +1155,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1486,18 +1505,19 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || - (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList()); - }); + (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1656,18 +1676,18 @@ public async Task SelectAndRefreshSingleFeatureFlag() var label1 = "App1_Label"; IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => - s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList()); - }); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => + s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1720,8 +1740,17 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetIfChanged); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetTestKey); string informationalInvocation = ""; string verboseInvocation = ""; @@ -1745,6 +1774,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1753,10 +1783,10 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "myFeature1", + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature2", value: @" { - ""id"": ""MyFeature"", + ""id"": ""MyFeature2"", ""description"": ""The new beta version of our web site."", ""display_name"": ""Beta Feature"", ""enabled"": true, @@ -1771,21 +1801,19 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); Assert.Null(config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); } [Fact] @@ -1796,8 +1824,11 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1821,6 +1852,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { @@ -1870,9 +1902,11 @@ public async Task MapTransformFeatureFlagWithRefresh() IConfigurationRefresher refresher = null; var featureFlags = new List { _kv }; var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1945,7 +1979,7 @@ public async Task MapTransformFeatureFlagWithRefresh() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index 06c88040..25edb546 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -505,9 +505,9 @@ public void DoesNotThrowKeyVaultExceptionWhenProviderIsOptional() .Returns(new MockAsyncPageable(new List { _kv })); var mockKeyValueAdapter = new Mock(MockBehavior.Strict); - mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(_kv)) + mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(It.IsAny())) .Returns(true); - mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(_kv, It.IsAny(), It.IsAny(), It.IsAny())) + mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Throws(new KeyVaultReferenceException("Key vault error", null)); mockKeyValueAdapter.Setup(adapter => adapter.OnChangeDetected(null)); mockKeyValueAdapter.Setup(adapter => adapter.OnConfigUpdated()); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 6edc1a9a..a7fdbd18 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -7,6 +7,7 @@ using Azure.Identity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Logging.Abstractions; using Moq; using System; @@ -1056,6 +1057,170 @@ public void RefreshTests_RefreshIsCancelled() Assert.Equal("TestValue1", config["TestKey1"]); } + [Fact] + public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var mockAsyncPageable = new MockAsyncPageable(_kvCollection); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Callback(() => mockAsyncPageable.UpdateCollection(_kvCollection)) + .Returns(mockAsyncPageable); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue3", config["TestKey3"]); + FirstKeyValue.Value = "newValue1"; + _kvCollection[2].Value = "newValue3"; + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("newValue3", config["TestKey3"]); + + _kvCollection.RemoveAt(2); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Null(config["TestKey3"]); + } + + [Fact] + public async Task RefreshTests_RegisterAllRefreshesFeatureFlags() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var featureFlags = new List { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""SuperUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) + }; + + var mockAsyncPageableKv = new MockAsyncPageable(_kvCollection); + + var mockAsyncPageableFf = new MockAsyncPageable(featureFlags); + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + mockAsyncPageableFf.UpdateCollection(featureFlags); + + return mockAsyncPageableFf; + } + + mockAsyncPageableKv.UpdateCollection(_kvCollection); + + return mockAsyncPageableKv; + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + options.UseFeatureFlags(); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newValue1"; + featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""AllUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newerValue1"; + featureFlags.RemoveAt(0); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newerValue1", config["TestKey1"]); + Assert.Null(config["FeatureManagement:MyFeature"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index bc7989b2..e3bf5500 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Core; +using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; using Microsoft.Extensions.Logging; @@ -10,6 +11,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -155,19 +157,69 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection; + private readonly List _collection = new List(); + private int _status; public MockAsyncPageable(List collection) { - _collection = collection; + foreach (ConfigurationSetting setting in collection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + + _status = 200; + } + + public void UpdateCollection(List newCollection) + { + if (_collection.Count() == newCollection.Count() && + _collection.All(setting => newCollection.Any(newSetting => + setting.Key == newSetting.Key && + setting.Value == newSetting.Value && + setting.Label == newSetting.Label && + setting.ETag == newSetting.ETag))) + { + _status = 304; + } + else + { + _status = 200; + + _collection.Clear(); + + foreach (ConfigurationSetting setting in newCollection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + } } #pragma warning disable 1998 public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) #pragma warning restore 1998 { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(_status)); + } + } + + internal class MockConfigurationSettingPageIterator : IConfigurationSettingPageIterator + { + public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(); + } + public IAsyncEnumerable> IteratePages(AsyncPageable pageable) + { + return pageable.AsPages(); } } @@ -182,7 +234,7 @@ public MockPageable(List collection) public override IEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } } From 47165a685e7a9694c36f93bb75af7e354ab7fd61 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 5 Feb 2025 16:19:53 -0800 Subject: [PATCH 10/22] Revert "Give the users the ability to have control over ConfigurationClient instance(s) used by the provider (#598)" This reverts commit 2ea9632420822845fd874cc90e6c2d0ff0a75f14. --- .../AzureAppConfigurationClientFactory.cs | 74 ------------------- .../AzureAppConfigurationOptions.cs | 17 ----- .../AzureAppConfigurationSource.cs | 32 ++++---- .../ConfigurationClientManager.cs | 64 +++++++++++++--- ...Configuration.AzureAppConfiguration.csproj | 1 - .../FailoverTests.cs | 19 ++--- 6 files changed, 79 insertions(+), 128 deletions(-) delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs deleted file mode 100644 index 6127822d..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -using Azure.Core; -using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Azure; -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class AzureAppConfigurationClientFactory : IAzureClientFactory - { - private readonly ConfigurationClientOptions _clientOptions; - - private readonly TokenCredential _credential; - private readonly IEnumerable _connectionStrings; - - public AzureAppConfigurationClientFactory( - IEnumerable connectionStrings, - ConfigurationClientOptions clientOptions) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - _connectionStrings = connectionStrings; - - _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); - } - - public AzureAppConfigurationClientFactory( - TokenCredential credential, - ConfigurationClientOptions clientOptions) - { - _credential = credential ?? throw new ArgumentNullException(nameof(credential)); - _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); - } - - public ConfigurationClient CreateClient(string endpoint) - { - if (string.IsNullOrEmpty(endpoint)) - { - throw new ArgumentNullException(nameof(endpoint)); - } - - if (!Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uriResult)) - { - throw new ArgumentException("Invalid host URI."); - } - - if (_credential != null) - { - return new ConfigurationClient(uriResult, _credential, _clientOptions); - } - - string connectionString = _connectionStrings.FirstOrDefault(cs => ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection) == endpoint); - - // - // falback to the first connection string - if (connectionString == null) - { - string id = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.IdSection); - string secret = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.SecretSection); - - connectionString = ConnectionStringUtils.Build(uriResult, id, secret); - } - - return new ConfigurationClient(connectionString, _clientOptions); - } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 0c80b200..7d9a9cad 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -3,7 +3,6 @@ // using Azure.Core; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; @@ -132,11 +131,6 @@ internal IEnumerable Adapters /// internal StartupOptions Startup { get; set; } = new StartupOptions(); - /// - /// Client factory that is responsible for creating instances of ConfigurationClient. - /// - internal IAzureClientFactory ClientFactory { get; private set; } - /// /// Initializes a new instance of the class. /// @@ -150,17 +144,6 @@ public AzureAppConfigurationOptions() }; } - /// - /// Sets the client factory used to create ConfigurationClient instances. - /// - /// The client factory. - /// The current instance. - public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) - { - ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); - return this; - } - /// /// Specify what key-values to include in the configuration provider. /// can be called multiple times to include multiple sets of key-values. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs index 83d20e2f..dee62006 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs @@ -1,11 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // -using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Azure; using System; -using System.Collections.Generic; -using System.Linq; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -33,33 +29,35 @@ public IConfigurationProvider Build(IConfigurationBuilder builder) try { AzureAppConfigurationOptions options = _optionsProvider(); + IConfigurationClientManager clientManager; if (options.ClientManager != null) { - return new AzureAppConfigurationProvider(options.ClientManager, options, _optional); + clientManager = options.ClientManager; } - - IEnumerable endpoints; - IAzureClientFactory clientFactory = options.ClientFactory; - - if (options.ConnectionStrings != null) + else if (options.ConnectionStrings != null) { - endpoints = options.ConnectionStrings.Select(cs => new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection))); - - clientFactory ??= new AzureAppConfigurationClientFactory(options.ConnectionStrings, options.ClientOptions); + clientManager = new ConfigurationClientManager( + options.ConnectionStrings, + options.ClientOptions, + options.ReplicaDiscoveryEnabled, + options.LoadBalancingEnabled); } else if (options.Endpoints != null && options.Credential != null) { - endpoints = options.Endpoints; - - clientFactory ??= new AzureAppConfigurationClientFactory(options.Credential, options.ClientOptions); + clientManager = new ConfigurationClientManager( + options.Endpoints, + options.Credential, + options.ClientOptions, + options.ReplicaDiscoveryEnabled, + options.LoadBalancingEnabled); } else { throw new ArgumentException($"Please call {nameof(AzureAppConfigurationOptions)}.{nameof(AzureAppConfigurationOptions.Connect)} to specify how to connect to Azure App Configuration."); } - provider = new AzureAppConfigurationProvider(new ConfigurationClientManager(clientFactory, endpoints, options.ReplicaDiscoveryEnabled, options.LoadBalancingEnabled), options, _optional); + provider = new AzureAppConfigurationProvider(clientManager, options, _optional); } catch (InvalidOperationException ex) // InvalidOperationException is thrown when any problems are found while configuring AzureAppConfigurationOptions or when SDK fails to create a configurationClient. { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs index 61840d03..a0215ca3 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs @@ -2,10 +2,10 @@ // Licensed under the MIT license. // +using Azure.Core; using Azure.Data.AppConfiguration; using DnsClient; using DnsClient.Protocol; -using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; @@ -26,11 +26,12 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration /// internal class ConfigurationClientManager : IConfigurationClientManager, IDisposable { - private readonly IAzureClientFactory _clientFactory; private readonly IList _clients; - private readonly Uri _endpoint; - + private readonly string _secret; + private readonly string _id; + private readonly TokenCredential _credential; + private readonly ConfigurationClientOptions _clientOptions; private readonly bool _replicaDiscoveryEnabled; private readonly SrvLookupClient _srvLookupClient; private readonly string _validDomain; @@ -51,20 +52,61 @@ internal class ConfigurationClientManager : IConfigurationClientManager, IDispos internal int RefreshClientsCalled { get; set; } = 0; public ConfigurationClientManager( - IAzureClientFactory clientFactory, - IEnumerable endpoints, + IEnumerable connectionStrings, + ConfigurationClientOptions clientOptions, bool replicaDiscoveryEnabled, bool loadBalancingEnabled) { - _clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory)); + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + string connectionString = connectionStrings.First(); + _endpoint = new Uri(ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.EndpointSection)); + _secret = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.SecretSection); + _id = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.IdSection); + _clientOptions = clientOptions; + _replicaDiscoveryEnabled = replicaDiscoveryEnabled; + + // If load balancing is enabled, shuffle the passed in connection strings to randomize the endpoint used on startup + if (loadBalancingEnabled) + { + connectionStrings = connectionStrings.ToList().Shuffle(); + } + + _validDomain = GetValidDomain(_endpoint); + _srvLookupClient = new SrvLookupClient(); + _clients = connectionStrings + .Select(cs => + { + var endpoint = new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection)); + return new ConfigurationClientWrapper(endpoint, new ConfigurationClient(cs, _clientOptions)); + }) + .ToList(); + } + + public ConfigurationClientManager( + IEnumerable endpoints, + TokenCredential credential, + ConfigurationClientOptions clientOptions, + bool replicaDiscoveryEnabled, + bool loadBalancingEnabled) + { if (endpoints == null || !endpoints.Any()) { throw new ArgumentNullException(nameof(endpoints)); } - _endpoint = endpoints.First(); + if (credential == null) + { + throw new ArgumentNullException(nameof(credential)); + } + _endpoint = endpoints.First(); + _credential = credential; + _clientOptions = clientOptions; _replicaDiscoveryEnabled = replicaDiscoveryEnabled; // If load balancing is enabled, shuffle the passed in endpoints to randomize the endpoint used on startup @@ -77,7 +119,7 @@ public ConfigurationClientManager( _srvLookupClient = new SrvLookupClient(); _clients = endpoints - .Select(endpoint => new ConfigurationClientWrapper(endpoint, clientFactory.CreateClient(endpoint.AbsoluteUri))) + .Select(endpoint => new ConfigurationClientWrapper(endpoint, new ConfigurationClient(endpoint, _credential, _clientOptions))) .ToList(); } @@ -247,7 +289,9 @@ private async Task RefreshFallbackClients(CancellationToken cancellationToken) { var targetEndpoint = new Uri($"https://{host}"); - ConfigurationClient configClient = _clientFactory.CreateClient(targetEndpoint.AbsoluteUri); + var configClient = _credential == null + ? new ConfigurationClient(ConnectionStringUtils.Build(targetEndpoint, _id, _secret), _clientOptions) + : new ConfigurationClient(targetEndpoint, _credential, _clientOptions); newDynamicClients.Add(new ConfigurationClientWrapper(targetEndpoint, configClient)); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index dc844599..3997c788 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -19,7 +19,6 @@ - diff --git a/tests/Tests.AzureAppConfiguration/FailoverTests.cs b/tests/Tests.AzureAppConfiguration/FailoverTests.cs index 929f9bef..86ea96b9 100644 --- a/tests/Tests.AzureAppConfiguration/FailoverTests.cs +++ b/tests/Tests.AzureAppConfiguration/FailoverTests.cs @@ -268,11 +268,10 @@ public void FailOverTests_AutoFailover() [Fact] public void FailOverTests_ValidateEndpoints() { - var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); - var configClientManager = new ConfigurationClientManager( - clientFactory, new[] { new Uri("https://foobar.azconfig.io") }, + new DefaultAzureCredential(), + new ConfigurationClientOptions(), true, false); @@ -286,8 +285,9 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager.IsValidEndpoint("azure.azconfig.bad.io")); var configClientManager2 = new ConfigurationClientManager( - clientFactory, new[] { new Uri("https://foobar.appconfig.azure.com") }, + new DefaultAzureCredential(), + new ConfigurationClientOptions(), true, false); @@ -301,8 +301,9 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager2.IsValidEndpoint("azure.appconfigbad.azure.com")); var configClientManager3 = new ConfigurationClientManager( - clientFactory, new[] { new Uri("https://foobar.azconfig-test.io") }, + new DefaultAzureCredential(), + new ConfigurationClientOptions(), true, false); @@ -310,8 +311,9 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager3.IsValidEndpoint("azure.azconfig.io")); var configClientManager4 = new ConfigurationClientManager( - clientFactory, new[] { new Uri("https://foobar.z1.appconfig-test.azure.com") }, + new DefaultAzureCredential(), + new ConfigurationClientOptions(), true, false); @@ -323,11 +325,10 @@ public void FailOverTests_ValidateEndpoints() [Fact] public void FailOverTests_GetNoDynamicClient() { - var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); - var configClientManager = new ConfigurationClientManager( - clientFactory, new[] { new Uri("https://azure.azconfig.io") }, + new DefaultAzureCredential(), + new ConfigurationClientOptions(), true, false); From c740e7ceb06caa0e587a6a8a0b146aede9844b31 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:51:56 -0800 Subject: [PATCH 11/22] Add `RegisterAll` API to enable monitoring collections of key-values for refresh (#574) * WIP * WIP testing out client extensions methods * WIP added selectors to multikeywatchers * remove unused property * WIP check for registerall changes to change refreshall * WIP * WIP fixing types and reslving errors * WIP fixing client extensions class * WIP * WIP update feature flag logic * WIP client extensions * WIP reload all flags on change * WIP * WIP fixing tests to return response for getconfigurationsettingsasync * WIP etag for tests * fix watchedcollections null * WIP tests, working for examples * remove unused variables * update to newest sdk version, remove unused * WIP fixing tests * WIP reworking testing to work with new etag approach * tests passing, fix mockasyncpageable * update sdk package version * fix loghelper, tests * WIP fixing aspages tests * revert watchesfeatureflags test * update test again * WIP * fixing watchconditions * separate selected key value collections from feature flag collections, separate selectors, add new methods to support new logic * comment and naming updates * fixing unit tests, namespace of defining/calling code needs to be same * fixing tests using AsPages * fix tests with pageablemanager * format * fix tests * fix tests * remove unused extension test class * fix comment, capitalization * check etag on 200, fix tests * add registerall test, fix refresh tests * fix condition for pages and old match conditions * WIP fixing PR comments, tests * check status after advancing existing etag enumerator * move around refresh logic * null check page etag, revert break to existing keys check in getrefreshedcollections * fix loadselected, replace selectedkvwatchers with registerall refresh time * fix comment in options * clean up tests * PR comments * PR comments * don't allow both registerall and register * fix check for calls to both register methods * PR comments for rename/small changes * fix compile error * simplify refreshasync path, fix naming from comments * remove redundant if check * simplify logic for minrefreshinterval * fix smaller comments * call loadselected when refreshing collection, separate data for individual refresh * in progress change to registerall include ff * fix load order * fix comments, rename logging constants to match new behavior * pr comments, refactor refreshasync * clean up etags dictionary creation * PR comments * add uncommitted changes to testhelper * update tests for registerall with feature flags, check ff keys to remove flags on refresh * PR comments * PR comments * use invalidoperationexception in configurerefresh, update loggingconstants to match behavior * remove unused changes --- .../AzureAppConfigurationOptions.cs | 113 ++- .../AzureAppConfigurationProvider.cs | 654 +++++++++++------- .../AzureAppConfigurationRefreshOptions.cs | 12 + .../ConfigurationSettingPageExtensions.cs | 33 + .../Constants/LoggingConstants.cs | 5 +- .../ConfigurationClientExtensions.cs | 129 +--- .../Extensions/StringExtensions.cs | 9 +- .../FeatureManagement/FeatureFlagOptions.cs | 3 +- .../GetKeyValueChangeCollectionOptions.cs | 13 - .../IConfigurationSettingPageIterator.cs | 13 + .../LogHelper.cs | 15 +- .../Models/KeyValueSelector.cs | 5 + .../Azure.Core.Testing/MockResponse.cs | 2 + .../FeatureManagementTests.cs | 100 ++- .../KeyVaultReferenceTests.cs | 4 +- .../RefreshTests.cs | 165 +++++ .../Tests.AzureAppConfiguration/TestHelper.cs | 60 +- 17 files changed, 865 insertions(+), 470 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7d9a9cad..9391f21b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -23,12 +23,13 @@ public class AzureAppConfigurationOptions private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); - private List _changeWatchers = new List(); - private List _multiKeyWatchers = new List(); + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors = new List(); + private List _selectors; private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; // The following set is sorted in descending order. // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. @@ -62,19 +63,29 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// A collection of . + /// A collection of specified by user. /// - internal IEnumerable KeyValueSelectors => _kvSelectors; + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } /// /// A collection of . /// - internal IEnumerable ChangeWatchers => _changeWatchers; + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable FeatureFlagWatchers => _ffWatchers; /// /// A collection of . @@ -96,11 +107,15 @@ internal IEnumerable Adapters internal IEnumerable KeyPrefixes => _keyPrefixes; /// - /// An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. /// - /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. /// @@ -142,6 +157,9 @@ public AzureAppConfigurationOptions() new JsonKeyValueAdapter(), new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } /// @@ -170,22 +188,30 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter throw new ArgumentNullException(nameof(keyFilter)); } + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + if (string.IsNullOrWhiteSpace(labelFilter)) { labelFilter = LabelFilter.Null; } - // Do not support * and , for label filter for now. - if (labelFilter.Contains('*') || labelFilter.Contains(',')) + if (!_selectCalled) { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + _selectors.Clear(); + + _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { KeyFilter = keyFilter, LabelFilter = labelFilter }); + return this; } @@ -201,7 +227,14 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) throw new ArgumentNullException(nameof(name)); } - _kvSelectors.AppendUnique(new KeyValueSelector + if (!_selectCalled) + { + _selectors.Clear(); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector { SnapshotName = name }); @@ -212,7 +245,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh on an individual flag level. + /// All loaded feature flags will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) @@ -237,25 +270,22 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = options.Label == null ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true }); } - foreach (var featureFlagSelector in options.FeatureFlagSelectors) + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - var featureFlagFilter = featureFlagSelector.KeyFilter; - var labelFilter = featureFlagSelector.LabelFilter; + _selectors.AppendUnique(featureFlagSelector); - Select(featureFlagFilter, labelFilter); - - _multiKeyWatchers.AppendUnique(new KeyValueWatcher + _ffWatchers.AppendUnique(new KeyValueWatcher { - Key = featureFlagFilter, - Label = labelFilter, + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); - } return this; @@ -376,18 +406,41 @@ public AzureAppConfigurationOptions ConfigureClientOptions(ActionA callback used to configure Azure App Configuration refresh options. public AzureAppConfigurationOptions ConfigureRefresh(Action configure) { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + var refreshOptions = new AzureAppConfigurationRefreshOptions(); configure?.Invoke(refreshOptions); - if (!refreshOptions.RefreshRegistrations.Any()) + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); + } + + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) { - throw new ArgumentException($"{nameof(ConfigureRefresh)}() must have at least one key-value registered for refresh."); + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); } - foreach (var item in refreshOptions.RefreshRegistrations) + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else { - item.RefreshInterval = refreshOptions.RefreshInterval; - _changeWatchers.Add(item); + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } } return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 89ba0510..b3756309 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -4,7 +4,6 @@ using Azure; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using Microsoft.Extensions.Logging; using System; @@ -32,9 +31,13 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); + private Dictionary _watchedIndividualKvs = new Dictionary(); + private HashSet _ffKeys = new HashSet(); + private Dictionary> _kvEtags = new Dictionary>(); + private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); + private DateTimeOffset _nextCollectionRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -108,11 +111,18 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); + IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (watchers.Any()) + bool hasWatchers = watchers.Any(); + TimeSpan minWatcherRefreshInterval = hasWatchers ? watchers.Min(w => w.RefreshInterval) : TimeSpan.MaxValue; + + if (options.RegisterAllEnabled) + { + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); + } + else if (hasWatchers) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + MinRefreshInterval = minWatcherRefreshInterval; } else { @@ -194,13 +204,15 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableMultiKeyWatchers = _options.MultiKeyWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); + bool isRefreshDue = utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableWatchers.Any() && - !refreshableMultiKeyWatchers.Any() && + !refreshableIndividualKvWatchers.Any() && + !refreshableFfWatchers.Any() && + !isRefreshDue && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -254,179 +266,166 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary watchedSettings = null; + Dictionary> kvEtags = null; + Dictionary> ffEtags = null; + HashSet ffKeys = null; + Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - List changedKeyValuesCollection = null; Dictionary data = null; + Dictionary ffCollectionData = null; + bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => - { - data = null; - watchedSettings = null; - keyValueChanges = new List(); - changedKeyValuesCollection = null; - refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); - logDebugBuilder.Clear(); - logInfoBuilder.Clear(); - - foreach (KeyValueWatcher changeWatcher in refreshableWatchers) - { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; - - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); - - KeyValueChange change = default; - - // - // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) - { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - else - { - // Load the key-value in case the previous load attempts had failed - - try - { - await CallWithRequestTracing( - async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) - { - watchedKv = null; - } - - if (watchedKv != null) - { - change = new KeyValueChange() - { - Key = watchedKv.Key, - Label = watchedKv.Label.NormalizeNull(), - Current = watchedKv, - ChangeType = KeyValueChangeType.Modified - }; - } - } - - // Check if a change has been detected in the key-value registered for refresh - if (change.ChangeType != KeyValueChangeType.None) - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); - keyValueChanges.Add(change); - - if (changeWatcher.RefreshAll) - { - refreshAll = true; - break; - } - } - else - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - } - } + { + kvEtags = null; + ffEtags = null; + ffKeys = null; + watchedIndividualKvs = null; + keyValueChanges = new List(); + data = null; + ffCollectionData = null; + ffCollectionUpdated = false; + refreshAll = false; + logDebugBuilder.Clear(); + logInfoBuilder.Clear(); + Uri endpoint = _configClientManager.GetEndpointForClient(client); - if (refreshAll) + if (_options.RegisterAllEnabled) + { + // Get key value collection changes if RegisterAll was called + if (isRefreshDue) { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; + refreshAll = await HaveCollectionsChanged( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector), + _kvEtags, + client, + cancellationToken).ConfigureAwait(false); } + } + else + { + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); + } - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + if (refreshAll) + { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true, + // or if any key-value collection change was detected. + kvEtags = new Dictionary>(); + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, ffKeys, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } - if (!changedKeyValuesCollection.Any()) + // Get feature flag changes + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - }, - cancellationToken) - .ConfigureAwait(false); + KeyFilter = watcher.Key, + LabelFilter = watcher.Label, + IsFeatureFlagSelector = true + }), + _ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) + { + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + ffKeys, + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + }, + cancellationToken) + .ConfigureAwait(false); - if (!refreshAll) + if (refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) + // Invalidate all the cached KeyVault secrets + foreach (IKeyValueAdapter adapter in _options.Adapters) { - UpdateNextRefreshTime(changeWatcher); + adapter.OnChangeDetected(); } - foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); - if (change.ChangeType == KeyValueChangeType.Modified) - { - ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + UpdateNextRefreshTime(changeWatcher); + } + } + else + { + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - foreach (Func> func in _options.Mappers) - { - setting = await func(setting).ConfigureAwait(false); - } + await ProcessKeyValueChangesAsync(keyValueChanges, _mappedData, watchedIndividualKvs).ConfigureAwait(false); - if (setting == null) - { - _mappedData.Remove(change.Key); - } - else - { - _mappedData[change.Key] = setting; - } - } - else if (change.ChangeType == KeyValueChangeType.Deleted) + if (ffCollectionUpdated) + { + // Remove all feature flag keys that are not present in the latest loading of feature flags, but were loaded previously + foreach (string key in _ffKeys.Except(ffKeys)) { - _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + _mappedData.Remove(key); } - // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting - foreach (IKeyValueAdapter adapter in _options.Adapters) + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + foreach (KeyValuePair kvp in mappedFfData) { - // If the current setting is null, try to pass the previous setting instead - if (change.Current != null) - { - adapter.OnChangeDetected(change.Current); - } - else if (change.Previous != null) - { - adapter.OnChangeDetected(change.Previous); - } + _mappedData[kvp.Key] = kvp.Value; } } - } - else - { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - - // Invalidate all the cached KeyVault secrets - foreach (IKeyValueAdapter adapter in _options.Adapters) - { - adapter.OnChangeDetected(); - } - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + // + // update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { UpdateNextRefreshTime(changeWatcher); } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || changedKeyValuesCollection?.Any() == true || keyValueChanges.Any()) + if (isRefreshDue) { - _watchedSettings = watchedSettings; + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) + { + _watchedIndividualKvs = watchedIndividualKvs ?? _watchedIndividualKvs; + + _ffEtags = ffEtags ?? _ffEtags; + + _kvEtags = kvEtags ?? _kvEtags; + + _ffKeys = ffKeys ?? _ffKeys; if (logDebugBuilder.Length > 0) { @@ -437,6 +436,7 @@ await CallWithRequestTracing( { _logger.LogInformation(logInfoBuilder.ToString().Trim()); } + // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. @@ -560,14 +560,21 @@ private void SetDirty(TimeSpan? maxDelay) { DateTimeOffset nextRefreshTime = AddRandomDelay(DateTimeOffset.UtcNow, maxDelay ?? DefaultMaxSetDirtyDelay); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + if (_options.RegisterAllEnabled) { - changeWatcher.NextRefreshTime = nextRefreshTime; + _nextCollectionRefreshTime = nextRefreshTime; + } + else + { + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) + { + kvWatcher.NextRefreshTime = nextRefreshTime; + } } - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { - changeWatcher.NextRefreshTime = nextRefreshTime; + featureFlagWatcher.NextRefreshTime = nextRefreshTime; } } @@ -717,34 +724,44 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary watchedSettings = null; + Dictionary> kvEtags = new Dictionary>(); + Dictionary> ffEtags = new Dictionary>(); + Dictionary watchedIndividualKvs = null; + HashSet ffKeys = new HashSet(); await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - data = await LoadSelectedKeyValues( + data = await LoadSelected( client, + kvEtags, + ffEtags, + _options.Selectors, + ffKeys, cancellationToken) .ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh( + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) .ConfigureAwait(false); - - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); }, cancellationToken) .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + if (_options.RegisterAllEnabled) + { + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + if (data != null) { // Invalidate all the cached KeyVault secrets @@ -754,51 +771,71 @@ await ExecuteWithFailOverPolicyAsync( } Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; + _mappedData = mappedData; + _kvEtags = kvEtags; + _ffEtags = ffEtags; + _watchedIndividualKvs = watchedIndividualKvs; + _ffKeys = ffKeys; } } - private async Task> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task> LoadSelected( + ConfigurationClient client, + Dictionary> kvEtags, + Dictionary> ffEtags, + IEnumerable selectors, + HashSet ffKeys, + CancellationToken cancellationToken) { - var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary data = new Dictionary(); - // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || - !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); - - if (useDefaultQuery) + foreach (KeyValueSelector loadOption in selectors) { - // Load all key-values with the null label. - var selector = new SettingSelector - { - KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null - }; - - await CallWithRequestTracing(async () => + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + var selector = new SettingSelector() { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); - } + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }; - foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) - { - IAsyncEnumerable settingsEnumerable; + var matchConditions = new List(); - if (string.IsNullOrEmpty(loadOption.SnapshotName)) - { - settingsEnumerable = client.GetConfigurationSettingsAsync( - new SettingSelector + await CallWithRequestTracing(async () => + { + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages(_options.ConfigurationSettingPageIterator).ConfigureAwait(false)) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }, - cancellationToken); + using Response response = page.GetRawResponse(); + + ETag serverEtag = (ETag)response.Headers.ETag; + + foreach (ConfigurationSetting setting in page.Values) + { + data[setting.Key] = setting; + + if (loadOption.IsFeatureFlagSelector) + { + ffKeys.Add(setting.Key); + } + } + + matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); + } + }).ConfigureAwait(false); + + if (loadOption.IsFeatureFlagSelector) + { + ffEtags[loadOption] = matchConditions; + } + else + { + kvEtags[loadOption] = matchConditions; + } } else { @@ -818,38 +855,42 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + IAsyncEnumerable settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( loadOption.SnapshotName, cancellationToken); - } - await CallWithRequestTracing(async () => - { - await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + await CallWithRequestTracing(async () => { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + { + data[setting.Key] = setting; + } + }).ConfigureAwait(false); + } } - return serverData; + return data; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh( + ConfigurationClient client, + IDictionary existingSettings, + CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + var watchedIndividualKvs = new Dictionary(); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -867,61 +908,84 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSettings; + return watchedIndividualKvs; } - private Dictionary UpdateWatchedKeyValueCollections(Dictionary watchedSettings, IDictionary existingSettings) + private async Task RefreshIndividualKvWatchers( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableIndividualKvWatchers, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher kvWatcher in refreshableIndividualKvWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; - foreach (ConfigurationSetting setting in currentKeyValues) + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + KeyValueChange change = default; + + // + // Find if there is a change associated with watcher + if (_watchedIndividualKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - } + else + { + // Load the key-value in case the previous load attempts had failed - return watchedSettings; - } + try + { + await CallWithRequestTracing( + async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) + { + watchedKv = null; + } - private async Task> GetRefreshedKeyValueCollections( - IEnumerable multiKeyWatchers, - ConfigurationClient client, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) - { - var keyValueChanges = new List(); + if (watchedKv != null) + { + change = new KeyValueChange() + { + Key = watchedKv.Key, + Label = watchedKv.Label.NormalizeNull(), + Current = watchedKv, + ChangeType = KeyValueChangeType.Modified + }; + } + } - foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) - { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, _watchedSettings.Values); + // Check if a change has been detected in the key-value registered for refresh + if (change.ChangeType != KeyValueChangeType.None) + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); + keyValueChanges.Add(change); - keyValueChanges.AddRange( - await client.GetKeyValueChangeCollection( - currentKeyValues, - new GetKeyValueChangeCollectionOptions - { - KeyFilter = changeWatcher.Key, - Label = changeWatcher.Label.NormalizeNull(), - RequestTracingEnabled = _requestTracingEnabled, - RequestTracingOptions = _requestTracingOptions - }, - logDebugBuilder, - logInfoBuilder, - endpoint, - cancellationToken) - .ConfigureAwait(false)); + if (kvWatcher.RefreshAll) + { + return true; + } + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + } } - return keyValueChanges; + return false; } private void SetData(IDictionary data) @@ -1188,30 +1252,6 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingSettings) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.StartsWith(keyPrefix) && kv.Label == label.NormalizeNull(); - }); - } - else - { - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.Equals(key) && kv.Label == label.NormalizeNull(); - }); - } - - return currentKeyValues; - } - private void EnsureAssemblyInspected() { if (!_isAssemblyInspected) @@ -1257,6 +1297,88 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } + private async Task HaveCollectionsChanged( + IEnumerable selectors, + Dictionary> pageEtags, + ConfigurationClient client, + CancellationToken cancellationToken) + { + bool haveCollectionsChanged = false; + + foreach (KeyValueSelector selector in selectors) + { + if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => haveCollectionsChanged = await client.HaveCollectionsChanged( + selector, + matchConditions, + _options.ConfigurationSettingPageIterator, + cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (haveCollectionsChanged) + { + return true; + } + } + + return haveCollectionsChanged; + } + + private async Task ProcessKeyValueChangesAsync( + IEnumerable keyValueChanges, + Dictionary mappedData, + Dictionary watchedIndividualKvs) + { + foreach (KeyValueChange change in keyValueChanges) + { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + + if (change.ChangeType == KeyValueChangeType.Modified) + { + ConfigurationSetting setting = change.Current; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + watchedIndividualKvs[changeIdentifier] = settingCopy; + + foreach (Func> func in _options.Mappers) + { + setting = await func(setting).ConfigureAwait(false); + } + + if (setting == null) + { + mappedData.Remove(change.Key); + } + else + { + mappedData[change.Key] = setting; + } + } + else if (change.ChangeType == KeyValueChangeType.Deleted) + { + mappedData.Remove(change.Key); + + watchedIndividualKvs.Remove(changeIdentifier); + } + + // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting + foreach (IKeyValueAdapter adapter in _options.Adapters) + { + // If the current setting is null, try to pass the previous setting instead + if (change.Current != null) + { + adapter.OnChangeDetected(change.Current); + } + else if (change.Previous != null) + { + adapter.OnChangeDetected(change.Previous); + } + } + } + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index f3fb6c4a..cf2847a8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -14,6 +14,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); + internal bool RegisterAllEnabled { get; private set; } /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. @@ -50,6 +51,17 @@ public AzureAppConfigurationRefreshOptions Register(string key, string label = L return this; } + /// + /// Register all key-values loaded outside of to be refreshed when the configuration provider's triggers a refresh. + /// The instance can be obtained by calling . + /// + public AzureAppConfigurationRefreshOptions RegisterAll() + { + RegisterAllEnabled = true; + + return this; + } + /// /// Sets the cache expiration time for the key-values registered for refresh. Default value is 30 seconds. Must be greater than 1 second. /// Any refresh operation triggered using will not update the value for a key until the cached value for that key has expired. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs new file mode 100644 index 00000000..aba7684b --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs @@ -0,0 +1,33 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 86576a48..3bdcaecd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -20,14 +20,15 @@ internal class LoggingConstants // Successful update, debug log level public const string RefreshKeyValueRead = "Key-value read from App Configuration."; public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; - public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValueCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; - public const string RefreshFeatureFlagUpdated = "Feature flag updated."; + public const string RefreshFeatureFlagsUpdated = "Feature flags reloaded."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags reloaded."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 4032b365..a50b4d45 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,12 +3,10 @@ // using Azure; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -66,131 +64,50 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetKeyValueChangeCollection( - this ConfigurationClient client, - IEnumerable keyValues, - GetKeyValueChangeCollectionOptions options, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IConfigurationSettingPageIterator pageIterator, CancellationToken cancellationToken) { - if (options == null) + if (matchConditions == null) { - throw new ArgumentNullException(nameof(options)); + throw new ArgumentNullException(nameof(matchConditions)); } - if (keyValues == null) + if (keyValueSelector == null) { - keyValues = Enumerable.Empty(); + throw new ArgumentNullException(nameof(keyValueSelector)); } - if (options.KeyFilter == null) + if (keyValueSelector.SnapshotName != null) { - options.KeyFilter = string.Empty; + throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); } - if (keyValues.Any(k => string.IsNullOrEmpty(k.Key))) + SettingSelector selector = new SettingSelector { - throw new ArgumentNullException($"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Key)}"); - } - - if (keyValues.Any(k => !string.Equals(k.Label.NormalizeNull(), options.Label.NormalizeNull()))) - { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Label)}"); - } - - if (keyValues.Any(k => k.Label != null && k.Label.Contains("*"))) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); - } - - var hasKeyValueCollectionChanged = false; - var selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label, - Fields = SettingFields.ETag | SettingFields.Key + KeyFilter = keyValueSelector.KeyFilter, + LabelFilter = keyValueSelector.LabelFilter }; - // Dictionary of eTags that we write to and use for comparison - var eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); + AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); - // Fetch e-tags for prefixed key-values that can be used to detect changes - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - hasKeyValueCollectionChanged = true; - break; - } + using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - // Check for any deletions - if (eTagMap.Any()) - { - hasKeyValueCollectionChanged = true; - } - - var changes = new List(); - - // If changes have been observed, refresh prefixed key-values - if (hasKeyValueCollectionChanged) + await foreach (Page page in pageable.AsPages(pageIterator, matchConditions).ConfigureAwait(false)) { - selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label - }; + using Response response = page.GetRawResponse(); - eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } + ETag serverEtag = (ETag)response.Headers.ETag; - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) + // Return true if the lists of etags are different + if ((!existingMatchConditionsEnumerator.MoveNext() || + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && + response.Status == (int)HttpStatusCode.OK) { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); + return true; } } - return changes; + // Need to check if pages were deleted and no change was found within the new shorter list of match conditions + return existingMatchConditionsEnumerator.MoveNext(); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 7bcf7212..e97dfa2f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -3,18 +3,11 @@ // namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { - internal static class LabelFilters - { - public static readonly string Null = "\0"; - - public static readonly string Any = "*"; - } - internal static class StringExtensions { public static string NormalizeNull(this string s) { - return s == LabelFilters.Null ? null : s; + return s == LabelFilter.Null ? null : s; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs index 1e8beae6..26390762 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs @@ -102,7 +102,8 @@ public FeatureFlagOptions Select(string featureFlagFilter, string labelFilter = FeatureFlagSelectors.AppendUnique(new KeyValueSelector { KeyFilter = featureFlagPrefix, - LabelFilter = labelFilter + LabelFilter = labelFilter, + IsFeatureFlagSelector = true }); return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs deleted file mode 100644 index 5cb9b83d..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class GetKeyValueChangeCollectionOptions - { - public string KeyFilter { get; set; } - public string Label { get; set; } - public bool RequestTracingEnabled { get; set; } - public RequestTracingOptions RequestTracingOptions { get; set; } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs new file mode 100644 index 00000000..08c95751 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs @@ -0,0 +1,13 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IConfigurationSettingPageIterator + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 11442dcb..4f999406 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -25,14 +25,19 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) return $"{LoggingConstants.RefreshFeatureFlagsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagReadMessage(string key, string label, string endpoint) - { - return $"{LoggingConstants.RefreshFeatureFlagRead} Key:'{key}' Label:'{label}' Endpoint:'{endpoint?.TrimEnd('/')}'"; + public static string BuildFeatureFlagsUpdatedMessage() + { + return LoggingConstants.RefreshFeatureFlagsUpdated; + } + + public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) + { + return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagUpdatedMessage(string key) + public static string BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage() { - return $"{LoggingConstants.RefreshFeatureFlagUpdated} Key:'{key}'"; + return LoggingConstants.RefreshSelectedKeyValuesAndFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 5491d04d..54bda1a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -24,6 +24,11 @@ public class KeyValueSelector /// public string SnapshotName { get; set; } + /// + /// A boolean that signifies whether this selector is intended to select feature flags. + /// + public bool IsFeatureFlagSelector { get; set; } + /// /// Determines whether the specified object is equal to the current object. /// diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77..aaee0b9c 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -17,6 +17,8 @@ public MockResponse(int status, string reasonPhrase = null) { Status = status; ReasonPhrase = reasonPhrase; + + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); } public override int Status { get; } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 96d2cf08..c374e2c3 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -659,19 +659,24 @@ public void UsesFeatureFlags() [Fact] public async Task WatchesFeatureFlags() { + var mockResponse = new MockResponse(200); + var featureFlags = new List { _kv }; - var mockResponse = new Mock(); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -711,7 +716,7 @@ public async Task WatchesFeatureFlags() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); featureFlags.Add(_kv2); @@ -730,11 +735,12 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -743,6 +749,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -801,17 +808,20 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -868,17 +878,20 @@ public async Task SkipRefreshIfCacheNotExpired() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -987,17 +1000,22 @@ public void QueriesFeatureFlags() } [Fact] - public async Task UsesEtagForFeatureFlagRefresh() + public async Task DoesNotUseEtagForFeatureFlagRefresh() { + var mockAsyncPageable = new MockAsyncPageable(new List { _kv }); + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(new List { _kv })); + .Callback(() => mockAsyncPageable.UpdateCollection(new List { _kv })) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1028,6 +1046,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1377,18 +1396,19 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || - (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList()); - }); + (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1547,18 +1567,18 @@ public async Task SelectAndRefreshSingleFeatureFlag() var label1 = "App1_Label"; IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => - s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList()); - }); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => + s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1611,8 +1631,17 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetIfChanged); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetTestKey); string informationalInvocation = ""; string verboseInvocation = ""; @@ -1636,6 +1665,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1644,10 +1674,10 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "myFeature1", + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature2", value: @" { - ""id"": ""MyFeature"", + ""id"": ""MyFeature2"", ""description"": ""The new beta version of our web site."", ""display_name"": ""Beta Feature"", ""enabled"": true, @@ -1662,21 +1692,19 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); Assert.Null(config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); } [Fact] @@ -1687,8 +1715,11 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1712,6 +1743,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { @@ -1761,9 +1793,11 @@ public async Task MapTransformFeatureFlagWithRefresh() IConfigurationRefresher refresher = null; var featureFlags = new List { _kv }; var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1836,7 +1870,7 @@ public async Task MapTransformFeatureFlagWithRefresh() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index ea520efc..274bfe7f 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -505,9 +505,9 @@ public void DoesNotThrowKeyVaultExceptionWhenProviderIsOptional() .Returns(new MockAsyncPageable(new List { _kv })); var mockKeyValueAdapter = new Mock(MockBehavior.Strict); - mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(_kv)) + mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(It.IsAny())) .Returns(true); - mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(_kv, It.IsAny(), It.IsAny(), It.IsAny())) + mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Throws(new KeyVaultReferenceException("Key vault error", null)); mockKeyValueAdapter.Setup(adapter => adapter.OnChangeDetected(null)); mockKeyValueAdapter.Setup(adapter => adapter.OnConfigUpdated()); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index dc904edc..e18e8453 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -7,6 +7,7 @@ using Azure.Identity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Logging.Abstractions; using Moq; using System; @@ -1052,6 +1053,170 @@ public void RefreshTests_RefreshIsCancelled() Assert.Equal("TestValue1", config["TestKey1"]); } + [Fact] + public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var mockAsyncPageable = new MockAsyncPageable(_kvCollection); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Callback(() => mockAsyncPageable.UpdateCollection(_kvCollection)) + .Returns(mockAsyncPageable); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue3", config["TestKey3"]); + FirstKeyValue.Value = "newValue1"; + _kvCollection[2].Value = "newValue3"; + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("newValue3", config["TestKey3"]); + + _kvCollection.RemoveAt(2); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Null(config["TestKey3"]); + } + + [Fact] + public async Task RefreshTests_RegisterAllRefreshesFeatureFlags() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var featureFlags = new List { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""SuperUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) + }; + + var mockAsyncPageableKv = new MockAsyncPageable(_kvCollection); + + var mockAsyncPageableFf = new MockAsyncPageable(featureFlags); + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + mockAsyncPageableFf.UpdateCollection(featureFlags); + + return mockAsyncPageableFf; + } + + mockAsyncPageableKv.UpdateCollection(_kvCollection); + + return mockAsyncPageableKv; + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + options.UseFeatureFlags(); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newValue1"; + featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""AllUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newerValue1"; + featureFlags.RemoveAt(0); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newerValue1", config["TestKey1"]); + Assert.Null(config["FeatureManagement:MyFeature"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index 75a23afe..ca929efc 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Core; +using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; using Microsoft.Extensions.Logging; @@ -10,6 +11,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -160,19 +162,69 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection; + private readonly List _collection = new List(); + private int _status; public MockAsyncPageable(List collection) { - _collection = collection; + foreach (ConfigurationSetting setting in collection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + + _status = 200; + } + + public void UpdateCollection(List newCollection) + { + if (_collection.Count() == newCollection.Count() && + _collection.All(setting => newCollection.Any(newSetting => + setting.Key == newSetting.Key && + setting.Value == newSetting.Value && + setting.Label == newSetting.Label && + setting.ETag == newSetting.ETag))) + { + _status = 304; + } + else + { + _status = 200; + + _collection.Clear(); + + foreach (ConfigurationSetting setting in newCollection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + } } #pragma warning disable 1998 public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) #pragma warning restore 1998 { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(_status)); + } + } + + internal class MockConfigurationSettingPageIterator : IConfigurationSettingPageIterator + { + public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(); + } + public IAsyncEnumerable> IteratePages(AsyncPageable pageable) + { + return pageable.AsPages(); } } @@ -187,7 +239,7 @@ public MockPageable(List collection) public override IEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } } From 6dc9ae2a42926952eb21339526264407bba5d5ee Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Thu, 6 Feb 2025 11:15:26 -0800 Subject: [PATCH 12/22] Give the users the ability to have control over ConfigurationClient instance(s) used by the provider (#598) (#617) * Introduced a new `AzureAppConfigurationClientFactory` class to handle the creation of `ConfigurationClient` instances * remove clients dictionary since we will not have hits and clients are already stored in ConfigurationClientManager * revert * add license + remove unused usings * ran dotnet format * add capability of fallback to different stores * add explicit type * address comments * remove scheme validation --------- Co-authored-by: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Co-authored-by: Sami Sadfa --- .../AzureAppConfigurationClientFactory.cs | 74 +++++++++++++++++++ .../AzureAppConfigurationOptions.cs | 17 +++++ .../AzureAppConfigurationSource.cs | 32 ++++---- .../ConfigurationClientManager.cs | 64 +++------------- ...Configuration.AzureAppConfiguration.csproj | 1 + .../FailoverTests.cs | 19 +++-- 6 files changed, 128 insertions(+), 79 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs new file mode 100644 index 00000000..6127822d --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Core; +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal class AzureAppConfigurationClientFactory : IAzureClientFactory + { + private readonly ConfigurationClientOptions _clientOptions; + + private readonly TokenCredential _credential; + private readonly IEnumerable _connectionStrings; + + public AzureAppConfigurationClientFactory( + IEnumerable connectionStrings, + ConfigurationClientOptions clientOptions) + { + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + _connectionStrings = connectionStrings; + + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public AzureAppConfigurationClientFactory( + TokenCredential credential, + ConfigurationClientOptions clientOptions) + { + _credential = credential ?? throw new ArgumentNullException(nameof(credential)); + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public ConfigurationClient CreateClient(string endpoint) + { + if (string.IsNullOrEmpty(endpoint)) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uriResult)) + { + throw new ArgumentException("Invalid host URI."); + } + + if (_credential != null) + { + return new ConfigurationClient(uriResult, _credential, _clientOptions); + } + + string connectionString = _connectionStrings.FirstOrDefault(cs => ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection) == endpoint); + + // + // falback to the first connection string + if (connectionString == null) + { + string id = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.IdSection); + string secret = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.SecretSection); + + connectionString = ConnectionStringUtils.Build(uriResult, id, secret); + } + + return new ConfigurationClient(connectionString, _clientOptions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 9391f21b..6e7f595a 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -3,6 +3,7 @@ // using Azure.Core; using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; @@ -146,6 +147,11 @@ internal IEnumerable Adapters /// internal StartupOptions Startup { get; set; } = new StartupOptions(); + /// + /// Client factory that is responsible for creating instances of ConfigurationClient. + /// + internal IAzureClientFactory ClientFactory { get; private set; } + /// /// Initializes a new instance of the class. /// @@ -162,6 +168,17 @@ public AzureAppConfigurationOptions() _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } + /// + /// Sets the client factory used to create ConfigurationClient instances. + /// + /// The client factory. + /// The current instance. + public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) + { + ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); + return this; + } + /// /// Specify what key-values to include in the configuration provider. /// can be called multiple times to include multiple sets of key-values. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs index dee62006..83d20e2f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using System; +using System.Collections.Generic; +using System.Linq; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -29,35 +33,33 @@ public IConfigurationProvider Build(IConfigurationBuilder builder) try { AzureAppConfigurationOptions options = _optionsProvider(); - IConfigurationClientManager clientManager; if (options.ClientManager != null) { - clientManager = options.ClientManager; + return new AzureAppConfigurationProvider(options.ClientManager, options, _optional); } - else if (options.ConnectionStrings != null) + + IEnumerable endpoints; + IAzureClientFactory clientFactory = options.ClientFactory; + + if (options.ConnectionStrings != null) { - clientManager = new ConfigurationClientManager( - options.ConnectionStrings, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.ConnectionStrings.Select(cs => new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection))); + + clientFactory ??= new AzureAppConfigurationClientFactory(options.ConnectionStrings, options.ClientOptions); } else if (options.Endpoints != null && options.Credential != null) { - clientManager = new ConfigurationClientManager( - options.Endpoints, - options.Credential, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.Endpoints; + + clientFactory ??= new AzureAppConfigurationClientFactory(options.Credential, options.ClientOptions); } else { throw new ArgumentException($"Please call {nameof(AzureAppConfigurationOptions)}.{nameof(AzureAppConfigurationOptions.Connect)} to specify how to connect to Azure App Configuration."); } - provider = new AzureAppConfigurationProvider(clientManager, options, _optional); + provider = new AzureAppConfigurationProvider(new ConfigurationClientManager(clientFactory, endpoints, options.ReplicaDiscoveryEnabled, options.LoadBalancingEnabled), options, _optional); } catch (InvalidOperationException ex) // InvalidOperationException is thrown when any problems are found while configuring AzureAppConfigurationOptions or when SDK fails to create a configurationClient. { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs index a0215ca3..61840d03 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs @@ -2,10 +2,10 @@ // Licensed under the MIT license. // -using Azure.Core; using Azure.Data.AppConfiguration; using DnsClient; using DnsClient.Protocol; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; @@ -26,12 +26,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration /// internal class ConfigurationClientManager : IConfigurationClientManager, IDisposable { + private readonly IAzureClientFactory _clientFactory; private readonly IList _clients; + private readonly Uri _endpoint; - private readonly string _secret; - private readonly string _id; - private readonly TokenCredential _credential; - private readonly ConfigurationClientOptions _clientOptions; + private readonly bool _replicaDiscoveryEnabled; private readonly SrvLookupClient _srvLookupClient; private readonly string _validDomain; @@ -52,61 +51,20 @@ internal class ConfigurationClientManager : IConfigurationClientManager, IDispos internal int RefreshClientsCalled { get; set; } = 0; public ConfigurationClientManager( - IEnumerable connectionStrings, - ConfigurationClientOptions clientOptions, - bool replicaDiscoveryEnabled, - bool loadBalancingEnabled) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - string connectionString = connectionStrings.First(); - _endpoint = new Uri(ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.EndpointSection)); - _secret = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.SecretSection); - _id = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.IdSection); - _clientOptions = clientOptions; - _replicaDiscoveryEnabled = replicaDiscoveryEnabled; - - // If load balancing is enabled, shuffle the passed in connection strings to randomize the endpoint used on startup - if (loadBalancingEnabled) - { - connectionStrings = connectionStrings.ToList().Shuffle(); - } - - _validDomain = GetValidDomain(_endpoint); - _srvLookupClient = new SrvLookupClient(); - - _clients = connectionStrings - .Select(cs => - { - var endpoint = new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection)); - return new ConfigurationClientWrapper(endpoint, new ConfigurationClient(cs, _clientOptions)); - }) - .ToList(); - } - - public ConfigurationClientManager( + IAzureClientFactory clientFactory, IEnumerable endpoints, - TokenCredential credential, - ConfigurationClientOptions clientOptions, bool replicaDiscoveryEnabled, bool loadBalancingEnabled) { + _clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory)); + if (endpoints == null || !endpoints.Any()) { throw new ArgumentNullException(nameof(endpoints)); } - if (credential == null) - { - throw new ArgumentNullException(nameof(credential)); - } - _endpoint = endpoints.First(); - _credential = credential; - _clientOptions = clientOptions; + _replicaDiscoveryEnabled = replicaDiscoveryEnabled; // If load balancing is enabled, shuffle the passed in endpoints to randomize the endpoint used on startup @@ -119,7 +77,7 @@ public ConfigurationClientManager( _srvLookupClient = new SrvLookupClient(); _clients = endpoints - .Select(endpoint => new ConfigurationClientWrapper(endpoint, new ConfigurationClient(endpoint, _credential, _clientOptions))) + .Select(endpoint => new ConfigurationClientWrapper(endpoint, clientFactory.CreateClient(endpoint.AbsoluteUri))) .ToList(); } @@ -289,9 +247,7 @@ private async Task RefreshFallbackClients(CancellationToken cancellationToken) { var targetEndpoint = new Uri($"https://{host}"); - var configClient = _credential == null - ? new ConfigurationClient(ConnectionStringUtils.Build(targetEndpoint, _id, _secret), _clientOptions) - : new ConfigurationClient(targetEndpoint, _credential, _clientOptions); + ConfigurationClient configClient = _clientFactory.CreateClient(targetEndpoint.AbsoluteUri); newDynamicClients.Add(new ConfigurationClientWrapper(targetEndpoint, configClient)); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index aeedd6e6..369f5dbd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -19,6 +19,7 @@ + diff --git a/tests/Tests.AzureAppConfiguration/FailoverTests.cs b/tests/Tests.AzureAppConfiguration/FailoverTests.cs index 86ea96b9..929f9bef 100644 --- a/tests/Tests.AzureAppConfiguration/FailoverTests.cs +++ b/tests/Tests.AzureAppConfiguration/FailoverTests.cs @@ -268,10 +268,11 @@ public void FailOverTests_AutoFailover() [Fact] public void FailOverTests_ValidateEndpoints() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -285,9 +286,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager.IsValidEndpoint("azure.azconfig.bad.io")); var configClientManager2 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.appconfig.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -301,9 +301,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager2.IsValidEndpoint("azure.appconfigbad.azure.com")); var configClientManager3 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig-test.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -311,9 +310,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager3.IsValidEndpoint("azure.azconfig.io")); var configClientManager4 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.z1.appconfig-test.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -325,10 +323,11 @@ public void FailOverTests_ValidateEndpoints() [Fact] public void FailOverTests_GetNoDynamicClient() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://azure.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); From 662250e2bba6f8bf505f31cfd3e6bce5f2ce978b Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Thu, 13 Feb 2025 10:01:57 -0800 Subject: [PATCH 13/22] update packages to 8.1.0 (#621) --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index d2ce2ed2..3392fea8 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.0.0 + 8.1.0 diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 4420db74..d9659c5f 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.0.0 + 8.1.0 diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 3997c788..9cac9155 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -35,7 +35,7 @@ - 8.0.0 + 8.1.0 From 7dc547c594f579d2c7755bba90b0babd6f182af0 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 19 Feb 2025 11:45:49 -0800 Subject: [PATCH 14/22] Bugfix for collection monitoring etag logic (#623) * check response code before comparing etags when they may be null * fix ordering of etag check and response check * fix mock response etag header * fix select in tests * add comment to explain loadselected etag is never null --- .../AzureAppConfigurationProvider.cs | 6 +++--- .../Extensions/ConfigurationClientExtensions.cs | 4 +--- .../Azure.Core.Testing/MockResponse.cs | 5 ++++- tests/Tests.AzureAppConfiguration/RefreshTests.cs | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b3756309..b509bd12 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -812,8 +812,6 @@ await CallWithRequestTracing(async () => { using Response response = page.GetRawResponse(); - ETag serverEtag = (ETag)response.Headers.ETag; - foreach (ConfigurationSetting setting in page.Values) { data[setting.Key] = setting; @@ -824,7 +822,9 @@ await CallWithRequestTracing(async () => } } - matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); + // The ETag will never be null here because it's not a conditional request + // Each successful response should have 200 status code and an ETag + matchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); } }).ConfigureAwait(false); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index a50b4d45..c4edfb0e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -95,11 +95,9 @@ public static async Task HaveCollectionsChanged(this ConfigurationClient c { using Response response = page.GetRawResponse(); - ETag serverEtag = (ETag)response.Headers.ETag; - // Return true if the lists of etags are different if ((!existingMatchConditionsEnumerator.MoveNext() || - !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag)) && response.Status == (int)HttpStatusCode.OK) { return true; diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index aaee0b9c..c60c2a25 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -18,7 +18,10 @@ public MockResponse(int status, string reasonPhrase = null) Status = status; ReasonPhrase = reasonPhrase; - AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); + if (status == 200) + { + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); + } } public override int Status { get; } diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index e18e8453..af6dd11d 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -1069,7 +1069,7 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.Select("TestKey*"); + options.Select("TestKey*", "label"); options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.ConfigureRefresh(refreshOptions => { @@ -1159,7 +1159,7 @@ MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.Select("TestKey*"); + options.Select("TestKey*", "label"); options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.ConfigureRefresh(refreshOptions => { From ab018909400be46a3fb64ed3d5b33436334b40e6 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:15:51 -0800 Subject: [PATCH 15/22] Add request tracing for push refresh usage (#626) * in progress * complete adding ispushrefreshused logic * rename pushrefresh tag string * remove empty line --- .../AzureAppConfigurationProvider.cs | 5 +++++ .../Constants/RequestTracingConstants.cs | 1 + .../RequestTracingOptions.cs | 5 +++++ .../TracingUtils.cs | 5 +++++ 4 files changed, 16 insertions(+) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b509bd12..d7d629f8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -548,6 +548,11 @@ public void ProcessPushNotification(PushNotification pushNotification, TimeSpan? if (_configClientManager.UpdateSyncToken(pushNotification.ResourceUri, pushNotification.SyncToken)) { + if (_requestTracingEnabled && _requestTracingOptions != null) + { + _requestTracingOptions.IsPushRefreshUsed = true; + } + SetDirty(maxDelay); } else diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs index 1084c274..f732ab95 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs @@ -33,6 +33,7 @@ internal class RequestTracingConstants public const string LoadBalancingEnabledTag = "LB"; public const string SignalRUsedTag = "SignalR"; public const string FailoverRequestTag = "Failover"; + public const string PushRefreshTag = "PushRefresh"; public const string FeatureFlagFilterTypeKey = "Filter"; public const string CustomFilter = "CSTM"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs index 3838ee95..bd8b7582 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs @@ -63,6 +63,11 @@ internal class RequestTracingOptions /// public bool IsFailoverRequest { get; set; } = false; + /// + /// Flag to indicate whether push refresh is used. + /// + public bool IsPushRefreshUsed { get; set; } = false; + /// /// Checks whether any tracing feature is used. /// diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs index b1b2b196..b3e12913 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs @@ -201,6 +201,11 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re correlationContextTags.Add(RequestTracingConstants.FailoverRequestTag); } + if (requestTracingOptions.IsPushRefreshUsed) + { + correlationContextTags.Add(RequestTracingConstants.PushRefreshTag); + } + var sb = new StringBuilder(); foreach (KeyValuePair kvp in correlationContextKeyValues) From 6ec290e527ed3934a8aec61063b8cdcd72459dea Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:50:40 -0800 Subject: [PATCH 16/22] Fix bug with feature flag and key-value select ordering (#629) * fix select order bug with feature flags * update to use default query static selector --- .../AzureAppConfigurationOptions.cs | 7 ++- .../FeatureManagementTests.cs | 57 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 9391f21b..a3dbfa05 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -22,6 +22,7 @@ public class AzureAppConfigurationOptions { private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); + private static readonly KeyValueSelector DefaultQuery = new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null }; private List _individualKvWatchers = new List(); private List _ffWatchers = new List(); @@ -159,7 +160,7 @@ public AzureAppConfigurationOptions() }; // Adds the default query to App Configuration if and are never called. - _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; + _selectors = new List { DefaultQuery }; } /// @@ -201,7 +202,7 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter if (!_selectCalled) { - _selectors.Clear(); + _selectors.Remove(DefaultQuery); _selectCalled = true; } @@ -229,7 +230,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) if (!_selectCalled) { - _selectors.Clear(); + _selectors.Remove(DefaultQuery); _selectCalled = true; } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index c374e2c3..d96e9e39 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -1066,6 +1066,63 @@ public void SelectFeatureFlags() Assert.Null(config["FeatureManagement:App2_Feature2"]); } + [Fact] + public void SelectOrderDoesNotAffectLoad() + { + var mockResponse = new Mock(); + var mockClient = new Mock(MockBehavior.Strict); + + List kvCollection = new List + { + ConfigurationModelFactory.ConfigurationSetting("TestKey1", "TestValue1", "label", + eTag: new ETag("0a76e3d7-7ec1-4e37-883c-9ea6d0d89e63")), + ConfigurationModelFactory.ConfigurationSetting("TestKey2", "TestValue2", "label", + eTag: new ETag("31c38369-831f-4bf1-b9ad-79db56c8b989")) + }; + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + List settingCollection; + + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + settingCollection = _featureFlagCollection; + } + else + { + settingCollection = kvCollection; + } + + var copy = new List(); + var newSetting = settingCollection.FirstOrDefault(s => (s.Key == selector.KeyFilter && s.Label == selector.LabelFilter)); + if (newSetting != null) + copy.Add(TestHelpers.CloneSetting(newSetting)); + return new MockAsyncPageable(copy); + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.UseFeatureFlags(ff => + { + ff.Select("App1_Feature1", "App1_Label"); + ff.Select("App2_Feature1", "App2_Label"); + }); + options.Select("TestKey1", "label"); + options.Select("TestKey2", "label"); + }) + .Build(); + + Assert.Equal("True", config["FeatureManagement:App1_Feature1"]); + Assert.Equal("False", config["FeatureManagement:App2_Feature1"]); + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue2", config["TestKey2"]); + } + [Fact] public void TestNullAndMissingValuesForConditions() { From 961cdebd2c379a03f715db28b3c5fa07f9772252 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:17:40 -0800 Subject: [PATCH 17/22] update packages to 8.1.1 (#630) --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index 3392fea8..4cd6bf4e 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.1.0 + 8.1.1 diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index d9659c5f..e327421b 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.1.0 + 8.1.1 diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 9cac9155..91b90bb1 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -35,7 +35,7 @@ - 8.1.0 + 8.1.1 From 4ab2dda65683af163f4309705e2309e092e22184 Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Wed, 12 Mar 2025 13:21:07 -0700 Subject: [PATCH 18/22] fix merge conflict errors --- .../AzureAppConfigurationProvider.cs | 7 ------- .../Extensions/StringExtensions.cs | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index bed5c70a..d7d629f8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -576,13 +576,6 @@ private void SetDirty(TimeSpan? maxDelay) kvWatcher.NextRefreshTime = nextRefreshTime; } } - else - { - foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) - { - kvWatcher.NextRefreshTime = nextRefreshTime; - } - } foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index a451ae88..61572030 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -11,5 +11,12 @@ public static string NormalizeNull(this string s) { return s == LabelFilter.Null ? null : s; } + + public static string ToBase64String(this string s) + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes(s); + + return Convert.ToBase64String(bytes); + } } } From c1cdea3cde95ac3844a71f7fe0c0798e3db7b487 Mon Sep 17 00:00:00 2001 From: Jimmy Campbell Date: Wed, 12 Mar 2025 16:47:24 -0400 Subject: [PATCH 19/22] Ensure kv collection refresh settings are not considered unless the feature is enabled. (#633) * Ensure that kv collection refresh interval is not used unless collection based refresh of key-values is enabled. Add tests to ensure that minimum refresh interval is respected for key-values and feature flags. * Remove duplicated tests. * fix. * Fix formatting. --- .../AzureAppConfigurationProvider.cs | 11 +++- .../RefreshTests.cs | 65 ++++++++++--------- .../Tests.AzureAppConfiguration/TestHelper.cs | 13 ++-- 3 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index d7d629f8..9d61faca 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -118,6 +118,13 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan if (options.RegisterAllEnabled) { + if (options.KvCollectionRefreshInterval <= TimeSpan.Zero) + { + throw new ArgumentException( + $"{nameof(options.KvCollectionRefreshInterval)} must be greater than zero seconds when using RegisterAll for refresh", + nameof(options)); + } + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); } else if (hasWatchers) @@ -206,7 +213,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) var utcNow = DateTimeOffset.UtcNow; IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); - bool isRefreshDue = utcNow >= _nextCollectionRefreshTime; + bool isRefreshDue = _options.RegisterAllEnabled && utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && @@ -412,7 +419,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => } } - if (isRefreshDue) + if (_options.RegisterAllEnabled && isRefreshDue) { _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index af6dd11d..e64b5184 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -212,7 +212,7 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -247,7 +247,7 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -284,7 +284,7 @@ public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -356,7 +356,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -430,7 +430,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -443,32 +443,33 @@ Response GetIfChanged(ConfigurationSetting setting, bool o } [Fact] - public async void RefreshTests_SingleServerCallOnSimultaneousMultipleRefresh() + public async Task RefreshTests_SingleServerCallOnSimultaneousMultipleRefresh() { var keyValueCollection = new List(_kvCollection); var requestCount = 0; var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + // Define delay for async operations + var operationDelay = TimeSpan.FromSeconds(6); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) .Returns(() => { requestCount++; - Thread.Sleep(6000); - var copy = new List(); foreach (var setting in keyValueCollection) { copy.Add(TestHelpers.CloneSetting(setting)); }; - return new MockAsyncPageable(copy); + return new MockAsyncPageable(copy, operationDelay); }); - Response GetIfChanged(ConfigurationSetting setting, bool onlyIfChanged, CancellationToken cancellationToken) + async Task> GetIfChanged(ConfigurationSetting setting, bool onlyIfChanged, CancellationToken cancellationToken) { requestCount++; - Thread.Sleep(6000); + await Task.Delay(operationDelay, cancellationToken); var newSetting = keyValueCollection.FirstOrDefault(s => s.Key == setting.Key && s.Label == setting.Label); var unchanged = (newSetting.Key == setting.Key && newSetting.Label == setting.Label && newSetting.Value == setting.Value); @@ -477,7 +478,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o } mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((Func>)GetIfChanged); + .Returns((Func>>)GetIfChanged); IConfigurationRefresher refresher = null; @@ -512,7 +513,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o } [Fact] - public void RefreshTests_RefreshAsyncThrowsOnRequestFailedException() + public async Task RefreshTests_RefreshAsyncThrowsOnRequestFailedException() { IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -539,7 +540,7 @@ public void RefreshTests_RefreshAsyncThrowsOnRequestFailedException() .Throws(new RequestFailedException("Request failed.")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); Action action = () => refresher.RefreshAsync().Wait(); Assert.Throws(action); @@ -575,7 +576,7 @@ public async Task RefreshTests_TryRefreshAsyncReturnsFalseOnRequestFailedExcepti .Throws(new RequestFailedException("Request failed.")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool result = await refresher.TryRefreshAsync(); Assert.False(result); @@ -608,7 +609,7 @@ public async Task RefreshTests_TryRefreshAsyncUpdatesConfigurationAndReturnsTrue _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool result = await refresher.TryRefreshAsync(); Assert.True(result); @@ -651,13 +652,13 @@ public async Task RefreshTests_TryRefreshAsyncReturnsFalseForAuthenticationFaile FirstKeyValue.Value = "newValue"; // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); // First call to GetConfigurationSettingAsync does not throw Assert.True(await refresher.TryRefreshAsync()); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); // Second call to GetConfigurationSettingAsync throws KeyVaultReferenceException Assert.False(await refresher.TryRefreshAsync()); @@ -704,7 +705,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await Assert.ThrowsAsync(async () => await refresher.RefreshAsync() @@ -748,7 +749,7 @@ public async Task RefreshTests_UpdatesAllSettingsIfInitialLoadFails() Assert.Null(configuration["TestKey3"]); // Make sure MinBackoffDuration has ended - Thread.Sleep(100); + await Task.Delay(100); // Act await Assert.ThrowsAsync(async () => @@ -763,7 +764,7 @@ await Assert.ThrowsAsync(async () => Assert.Null(configuration["TestKey3"]); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -825,7 +826,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o keyValueCollection = keyValueCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool firstRefreshResult = await refresher.TryRefreshAsync(); Assert.False(firstRefreshResult); @@ -835,7 +836,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue3", config["TestKey3"]); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool secondRefreshResult = await refresher.TryRefreshAsync(); Assert.True(secondRefreshResult); @@ -876,7 +877,7 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -917,7 +918,7 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi _kvCollection[_kvCollection.IndexOf(refreshRegisteredSetting)] = TestHelpers.ChangeValue(refreshRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -959,7 +960,7 @@ public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() _kvCollection[_kvCollection.IndexOf(refreshAllRegisteredSetting)] = TestHelpers.ChangeValue(refreshAllRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -1020,7 +1021,7 @@ public void RefreshTests_ConfigureRefreshThrowsOnNoRegistration() } [Fact] - public void RefreshTests_RefreshIsCancelled() + public async Task RefreshTests_RefreshIsCancelled() { IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -1043,7 +1044,7 @@ public void RefreshTests_RefreshIsCancelled() FirstKeyValue.Value = "newValue1"; // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); using var cancellationSource = new CancellationTokenSource(); cancellationSource.Cancel(); @@ -1087,7 +1088,7 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() _kvCollection[2].Value = "newValue3"; // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -1097,7 +1098,7 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() _kvCollection.RemoveAt(2); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -1198,7 +1199,7 @@ MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -1209,7 +1210,7 @@ MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) featureFlags.RemoveAt(0); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index ca929efc..9fd3f388 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -164,8 +164,9 @@ class MockAsyncPageable : AsyncPageable { private readonly List _collection = new List(); private int _status; + private readonly TimeSpan? _delay; - public MockAsyncPageable(List collection) + public MockAsyncPageable(List collection, TimeSpan? delay = null) { foreach (ConfigurationSetting setting in collection) { @@ -177,6 +178,7 @@ public MockAsyncPageable(List collection) } _status = 200; + _delay = delay; } public void UpdateCollection(List newCollection) @@ -207,10 +209,13 @@ public void UpdateCollection(List newCollection) } } -#pragma warning disable 1998 - public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) -#pragma warning restore 1998 + public override async IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { + if (_delay.HasValue) + { + await Task.Delay(_delay.Value); + } + yield return Page.FromValues(_collection, null, new MockResponse(_status)); } } From c2e3558e5dd1eb27fcfb44aec15e8486ad8dc50c Mon Sep 17 00:00:00 2001 From: Sami Sadfa Date: Tue, 18 Mar 2025 12:45:53 -0700 Subject: [PATCH 20/22] revert versions --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index 4cd6bf4e..a5d4dee4 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.1.1 + 8.1.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index e327421b..4354b77e 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.1.1 + 8.1.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 5b941baa..369f5dbd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -36,7 +36,7 @@ - 8.1.1 + 8.1.0-preview From 87f0f85ca2e4011f82d93a864e35c6c804cd6c39 Mon Sep 17 00:00:00 2001 From: Richard chen <99175581+RichardChen820@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:49:46 -0700 Subject: [PATCH 21/22] Shorten the defeult timeout of individual call to backend (#620) Co-authored-by: AMER JUSUPOVIC --- .../AzureAppConfigurationOptions.cs | 1035 +++++++++-------- .../AzureAppConfigurationProvider.cs | 7 + 2 files changed, 528 insertions(+), 514 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index a3dbfa05..db3b6c3d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -1,517 +1,524 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -using Azure.Core; -using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - /// - /// Options used to configure the behavior of an Azure App Configuration provider. - /// If neither nor is ever called, all key-values with no label are included in the configuration provider. - /// - public class AzureAppConfigurationOptions - { - private const int MaxRetries = 2; - private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + /// + /// Options used to configure the behavior of an Azure App Configuration provider. + /// If neither nor is ever called, all key-values with no label are included in the configuration provider. + /// + public class AzureAppConfigurationOptions + { + private const int MaxRetries = 2; + private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); + private static readonly TimeSpan NetworkTimeout = TimeSpan.FromSeconds(10); private static readonly KeyValueSelector DefaultQuery = new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null }; - - private List _individualKvWatchers = new List(); - private List _ffWatchers = new List(); - private List _adapters; - private List>> _mappers = new List>>(); - private List _selectors; - private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); - private bool _selectCalled = false; - - // The following set is sorted in descending order. - // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. - private SortedSet _keyPrefixes = new SortedSet(Comparer.Create((k1, k2) => -string.Compare(k1, k2, StringComparison.OrdinalIgnoreCase))); - - /// - /// Flag to indicate whether replica discovery is enabled. - /// - public bool ReplicaDiscoveryEnabled { get; set; } = true; - - /// - /// Flag to indicate whether load balancing is enabled. - /// - public bool LoadBalancingEnabled { get; set; } - - /// - /// The list of connection strings used to connect to an Azure App Configuration store and its replicas. - /// - internal IEnumerable ConnectionStrings { get; private set; } - - /// - /// The list of endpoints of an Azure App Configuration store. - /// If this property is set, the property also needs to be set. - /// - internal IEnumerable Endpoints { get; private set; } - - /// - /// The credential used to connect to the Azure App Configuration. - /// If this property is set, the property also needs to be set. - /// - internal TokenCredential Credential { get; private set; } - - /// - /// A collection of specified by user. - /// - internal IEnumerable Selectors => _selectors; - - /// - /// Indicates if was called. - /// - internal bool RegisterAllEnabled { get; private set; } - - /// - /// Refresh interval for selected key-value collections when is called. - /// - internal TimeSpan KvCollectionRefreshInterval { get; private set; } - - /// - /// A collection of . - /// - internal IEnumerable IndividualKvWatchers => _individualKvWatchers; - - /// - /// A collection of . - /// - internal IEnumerable FeatureFlagWatchers => _ffWatchers; - - /// - /// A collection of . - /// - internal IEnumerable Adapters - { - get => _adapters; - set => _adapters = value?.ToList(); - } - - /// - /// A collection of user defined functions that transform each . - /// - internal IEnumerable>> Mappers => _mappers; - - /// - /// A collection of key prefixes to be trimmed. - /// - internal IEnumerable KeyPrefixes => _keyPrefixes; - - /// - /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. - /// - internal IConfigurationClientManager ClientManager { get; set; } - - /// - /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. - /// - internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } - - /// - /// An optional timespan value to set the minimum backoff duration to a value other than the default. - /// - internal TimeSpan MinBackoffDuration { get; set; } = FailOverConstants.MinBackoffDuration; - - /// - /// Options used to configure the client used to communicate with Azure App Configuration. - /// - internal ConfigurationClientOptions ClientOptions { get; } = GetDefaultClientOptions(); - - /// - /// Flag to indicate whether Key Vault options have been configured. - /// - internal bool IsKeyVaultConfigured { get; private set; } = false; - - /// - /// Flag to indicate whether Key Vault secret values will be refreshed automatically. - /// - internal bool IsKeyVaultRefreshConfigured { get; private set; } = false; - - /// - /// Indicates all types of feature filters used by the application. - /// - internal FeatureFlagTracing FeatureFlagTracing { get; set; } = new FeatureFlagTracing(); - - /// - /// Options used to configure provider startup. - /// - internal StartupOptions Startup { get; set; } = new StartupOptions(); - - /// - /// Initializes a new instance of the class. - /// - public AzureAppConfigurationOptions() - { - _adapters = new List() + + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); + private List _adapters; + private List>> _mappers = new List>>(); + private List _selectors; + private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; + + // The following set is sorted in descending order. + // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. + private SortedSet _keyPrefixes = new SortedSet(Comparer.Create((k1, k2) => -string.Compare(k1, k2, StringComparison.OrdinalIgnoreCase))); + + /// + /// Flag to indicate whether replica discovery is enabled. + /// + public bool ReplicaDiscoveryEnabled { get; set; } = true; + + /// + /// Flag to indicate whether load balancing is enabled. + /// + public bool LoadBalancingEnabled { get; set; } + + /// + /// The list of connection strings used to connect to an Azure App Configuration store and its replicas. + /// + internal IEnumerable ConnectionStrings { get; private set; } + + /// + /// The list of endpoints of an Azure App Configuration store. + /// If this property is set, the property also needs to be set. + /// + internal IEnumerable Endpoints { get; private set; } + + /// + /// The credential used to connect to the Azure App Configuration. + /// If this property is set, the property also needs to be set. + /// + internal TokenCredential Credential { get; private set; } + + /// + /// A collection of specified by user. + /// + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } + + /// + /// A collection of . + /// + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; + + /// + /// A collection of . + /// + internal IEnumerable FeatureFlagWatchers => _ffWatchers; + + /// + /// A collection of . + /// + internal IEnumerable Adapters + { + get => _adapters; + set => _adapters = value?.ToList(); + } + + /// + /// A collection of user defined functions that transform each . + /// + internal IEnumerable>> Mappers => _mappers; + + /// + /// A collection of key prefixes to be trimmed. + /// + internal IEnumerable KeyPrefixes => _keyPrefixes; + + /// + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// + internal IConfigurationClientManager ClientManager { get; set; } + + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + + /// + /// An optional timespan value to set the minimum backoff duration to a value other than the default. + /// + internal TimeSpan MinBackoffDuration { get; set; } = FailOverConstants.MinBackoffDuration; + + /// + /// Options used to configure the client used to communicate with Azure App Configuration. + /// + internal ConfigurationClientOptions ClientOptions { get; } = GetDefaultClientOptions(); + + /// + /// Flag to indicate whether Key Vault options have been configured. + /// + internal bool IsKeyVaultConfigured { get; private set; } = false; + + /// + /// Flag to indicate whether Key Vault secret values will be refreshed automatically. + /// + internal bool IsKeyVaultRefreshConfigured { get; private set; } = false; + + /// + /// Indicates all types of feature filters used by the application. + /// + internal FeatureFlagTracing FeatureFlagTracing { get; set; } = new FeatureFlagTracing(); + + /// + /// Options used to configure provider startup. + /// + internal StartupOptions Startup { get; set; } = new StartupOptions(); + + /// + /// Initializes a new instance of the class. + /// + public AzureAppConfigurationOptions() + { + _adapters = new List() + { + new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), + new JsonKeyValueAdapter(), + new FeatureManagementKeyValueAdapter(FeatureFlagTracing) + }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { DefaultQuery }; + } + + /// + /// Specify what key-values to include in the configuration provider. + /// can be called multiple times to include multiple sets of key-values. + /// + /// + /// The key filter to apply when querying Azure App Configuration for key-values. + /// An asterisk (*) can be added to the end to return all key-values whose key begins with the key filter. + /// e.g. key filter `abc*` returns all key-values whose key starts with `abc`. + /// A comma (,) can be used to select multiple key-values. Comma separated filters must exactly match a key to select it. + /// Using asterisk to select key-values that begin with a key filter while simultaneously using comma separated key filters is not supported. + /// E.g. the key filter `abc*,def` is not supported. The key filters `abc*` and `abc,def` are supported. + /// For all other cases the characters: asterisk (*), comma (,), and backslash (\) are reserved. Reserved characters must be escaped using a backslash (\). + /// e.g. the key filter `a\\b\,\*c*` returns all key-values whose key starts with `a\b,*c`. + /// Built-in key filter options: . + /// + /// + /// The label filter to apply when querying Azure App Configuration for key-values. By default the null label will be used. Built-in label filter options: + /// The characters asterisk (*) and comma (,) are not supported. Backslash (\) character is reserved and must be escaped using another backslash (\). + /// + public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter = LabelFilter.Null) + { + if (string.IsNullOrEmpty(keyFilter)) + { + throw new ArgumentNullException(nameof(keyFilter)); + } + + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + + if (string.IsNullOrWhiteSpace(labelFilter)) + { + labelFilter = LabelFilter.Null; + } + + if (!_selectCalled) + { + _selectors.Remove(DefaultQuery); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector + { + KeyFilter = keyFilter, + LabelFilter = labelFilter + }); + + return this; + } + + /// + /// Specify a snapshot and include its contained key-values in the configuration provider. + /// can be called multiple times to include key-values from multiple snapshots. + /// + /// The name of the snapshot in Azure App Configuration. + public AzureAppConfigurationOptions SelectSnapshot(string name) + { + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name)); + } + + if (!_selectCalled) + { + _selectors.Remove(DefaultQuery); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector + { + SnapshotName = name + }); + + return this; + } + + /// + /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. + /// If no filtering is specified via the then all feature flags with no label are loaded. + /// All loaded feature flags will be automatically registered for refresh as a collection. + /// + /// A callback used to configure feature flag options. + public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) + { + FeatureFlagOptions options = new FeatureFlagOptions(); + configure?.Invoke(options); + + if (options.RefreshInterval < RefreshConstants.MinimumFeatureFlagRefreshInterval) + { + throw new ArgumentOutOfRangeException(nameof(options.RefreshInterval), options.RefreshInterval.TotalMilliseconds, + string.Format(ErrorMessages.RefreshIntervalTooShort, RefreshConstants.MinimumFeatureFlagRefreshInterval.TotalMilliseconds)); + } + + if (options.FeatureFlagSelectors.Count() != 0 && options.Label != null) + { + throw new InvalidOperationException($"Please select feature flags by either the {nameof(options.Select)} method or by setting the {nameof(options.Label)} property, not both."); + } + + if (options.FeatureFlagSelectors.Count() == 0) + { + // Select clause is not present + options.FeatureFlagSelectors.Add(new KeyValueSelector + { + KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true + }); + } + + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) + { + _selectors.AppendUnique(featureFlagSelector); + + _ffWatchers.AppendUnique(new KeyValueWatcher + { + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, + // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins + RefreshInterval = options.RefreshInterval + }); + } + + return this; + } + + /// + /// Connect the provider to the Azure App Configuration service via a connection string. + /// + /// + /// Used to authenticate with Azure App Configuration. + /// + public AzureAppConfigurationOptions Connect(string connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentNullException(nameof(connectionString)); + } + + return Connect(new List { connectionString }); + } + + /// + /// Connect the provider to an Azure App Configuration store and its replicas via a list of connection strings. + /// + /// + /// Used to authenticate with Azure App Configuration. + /// + public AzureAppConfigurationOptions Connect(IEnumerable connectionStrings) + { + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + if (connectionStrings.Distinct().Count() != connectionStrings.Count()) + { + throw new ArgumentException($"All values in '{nameof(connectionStrings)}' must be unique."); + } + + Endpoints = null; + Credential = null; + ConnectionStrings = connectionStrings; + return this; + } + + /// + /// Connect the provider to Azure App Configuration using endpoint and token credentials. + /// + /// The endpoint of the Azure App Configuration to connect to. + /// Token credentials to use to connect. + public AzureAppConfigurationOptions Connect(Uri endpoint, TokenCredential credential) + { + if (endpoint == null) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + if (credential == null) + { + throw new ArgumentNullException(nameof(credential)); + } + + return Connect(new List() { endpoint }, credential); + } + + /// + /// Connect the provider to an Azure App Configuration store and its replicas using a list of endpoints and a token credential. + /// + /// The list of endpoints of an Azure App Configuration store and its replicas to connect to. + /// Token credential to use to connect. + public AzureAppConfigurationOptions Connect(IEnumerable endpoints, TokenCredential credential) + { + if (endpoints == null || !endpoints.Any()) + { + throw new ArgumentNullException(nameof(endpoints)); + } + + if (endpoints.Distinct(new EndpointComparer()).Count() != endpoints.Count()) + { + throw new ArgumentException($"All values in '{nameof(endpoints)}' must be unique."); + } + + Credential = credential ?? throw new ArgumentNullException(nameof(credential)); + + Endpoints = endpoints; + ConnectionStrings = null; + return this; + } + + /// + /// Trims the provided prefix from the keys of all key-values retrieved from Azure App Configuration. + /// + /// The prefix to be trimmed. + public AzureAppConfigurationOptions TrimKeyPrefix(string prefix) + { + if (string.IsNullOrEmpty(prefix)) + { + throw new ArgumentNullException(nameof(prefix)); + } + + _keyPrefixes.Add(prefix); + return this; + } + + /// + /// Configure the client(s) used to communicate with Azure App Configuration. + /// + /// A callback used to configure Azure App Configuration client options. + public AzureAppConfigurationOptions ConfigureClientOptions(Action configure) + { + configure?.Invoke(ClientOptions); + return this; + } + + /// + /// Configure refresh for key-values in the configuration provider. + /// + /// A callback used to configure Azure App Configuration refresh options. + public AzureAppConfigurationOptions ConfigureRefresh(Action configure) + { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + + var refreshOptions = new AzureAppConfigurationRefreshOptions(); + configure?.Invoke(refreshOptions); + + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); + } + + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) + { + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); + } + + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else + { + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } + } + + return this; + } + + /// + /// Get an instance of that can be used to trigger a refresh for the registered key-values. + /// + /// An instance of . + public IConfigurationRefresher GetRefresher() + { + return _refresher; + } + + /// + /// Configures the Azure App Configuration provider to use the provided Key Vault configuration to resolve key vault references. + /// + /// A callback used to configure Azure App Configuration key vault options. + public AzureAppConfigurationOptions ConfigureKeyVault(Action configure) + { + var keyVaultOptions = new AzureAppConfigurationKeyVaultOptions(); + configure?.Invoke(keyVaultOptions); + + if (keyVaultOptions.Credential != null && keyVaultOptions.SecretResolver != null) + { + throw new InvalidOperationException($"Cannot configure both default credentials and secret resolver for Key Vault references. Please call either {nameof(keyVaultOptions.SetCredential)} or {nameof(keyVaultOptions.SetSecretResolver)} method, not both."); + } + + _adapters.RemoveAll(a => a is AzureKeyVaultKeyValueAdapter); + _adapters.Add(new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider(keyVaultOptions))); + + IsKeyVaultRefreshConfigured = keyVaultOptions.IsKeyVaultRefreshConfigured; + IsKeyVaultConfigured = true; + return this; + } + + /// + /// Provides a way to transform settings retrieved from App Configuration before they are processed by the configuration provider. + /// + /// A callback registered by the user to transform each configuration setting. + public AzureAppConfigurationOptions Map(Func> mapper) + { + if (mapper == null) + { + throw new ArgumentNullException(nameof(mapper)); + } + + _mappers.Add(mapper); + return this; + } + + /// + /// Configure the provider behavior when loading data from Azure App Configuration on startup. + /// + /// A callback used to configure Azure App Configuration startup options. + public AzureAppConfigurationOptions ConfigureStartupOptions(Action configure) + { + configure?.Invoke(Startup); + return this; + } + + private static ConfigurationClientOptions GetDefaultClientOptions() + { + var clientOptions = new ConfigurationClientOptions(ConfigurationClientOptions.ServiceVersion.V2023_10_01); + clientOptions.Retry.MaxRetries = MaxRetries; + clientOptions.Retry.MaxDelay = MaxRetryDelay; + clientOptions.Retry.Mode = RetryMode.Exponential; + clientOptions.AddPolicy(new UserAgentHeaderPolicy(), HttpPipelinePosition.PerCall); + clientOptions.Transport = new HttpClientTransport(new HttpClient() { - new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), - new JsonKeyValueAdapter(), - new FeatureManagementKeyValueAdapter(FeatureFlagTracing) - }; - - // Adds the default query to App Configuration if and are never called. - _selectors = new List { DefaultQuery }; - } - - /// - /// Specify what key-values to include in the configuration provider. - /// can be called multiple times to include multiple sets of key-values. - /// - /// - /// The key filter to apply when querying Azure App Configuration for key-values. - /// An asterisk (*) can be added to the end to return all key-values whose key begins with the key filter. - /// e.g. key filter `abc*` returns all key-values whose key starts with `abc`. - /// A comma (,) can be used to select multiple key-values. Comma separated filters must exactly match a key to select it. - /// Using asterisk to select key-values that begin with a key filter while simultaneously using comma separated key filters is not supported. - /// E.g. the key filter `abc*,def` is not supported. The key filters `abc*` and `abc,def` are supported. - /// For all other cases the characters: asterisk (*), comma (,), and backslash (\) are reserved. Reserved characters must be escaped using a backslash (\). - /// e.g. the key filter `a\\b\,\*c*` returns all key-values whose key starts with `a\b,*c`. - /// Built-in key filter options: . - /// - /// - /// The label filter to apply when querying Azure App Configuration for key-values. By default the null label will be used. Built-in label filter options: - /// The characters asterisk (*) and comma (,) are not supported. Backslash (\) character is reserved and must be escaped using another backslash (\). - /// - public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter = LabelFilter.Null) - { - if (string.IsNullOrEmpty(keyFilter)) - { - throw new ArgumentNullException(nameof(keyFilter)); - } - - // Do not support * and , for label filter for now. - if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) - { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); - } - - if (string.IsNullOrWhiteSpace(labelFilter)) - { - labelFilter = LabelFilter.Null; - } - - if (!_selectCalled) - { - _selectors.Remove(DefaultQuery); - - _selectCalled = true; - } - - _selectors.AppendUnique(new KeyValueSelector - { - KeyFilter = keyFilter, - LabelFilter = labelFilter - }); - - return this; - } - - /// - /// Specify a snapshot and include its contained key-values in the configuration provider. - /// can be called multiple times to include key-values from multiple snapshots. - /// - /// The name of the snapshot in Azure App Configuration. - public AzureAppConfigurationOptions SelectSnapshot(string name) - { - if (string.IsNullOrEmpty(name)) - { - throw new ArgumentNullException(nameof(name)); - } - - if (!_selectCalled) - { - _selectors.Remove(DefaultQuery); - - _selectCalled = true; - } - - _selectors.AppendUnique(new KeyValueSelector - { - SnapshotName = name + Timeout = NetworkTimeout }); - - return this; - } - - /// - /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. - /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh as a collection. - /// - /// A callback used to configure feature flag options. - public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) - { - FeatureFlagOptions options = new FeatureFlagOptions(); - configure?.Invoke(options); - - if (options.RefreshInterval < RefreshConstants.MinimumFeatureFlagRefreshInterval) - { - throw new ArgumentOutOfRangeException(nameof(options.RefreshInterval), options.RefreshInterval.TotalMilliseconds, - string.Format(ErrorMessages.RefreshIntervalTooShort, RefreshConstants.MinimumFeatureFlagRefreshInterval.TotalMilliseconds)); - } - - if (options.FeatureFlagSelectors.Count() != 0 && options.Label != null) - { - throw new InvalidOperationException($"Please select feature flags by either the {nameof(options.Select)} method or by setting the {nameof(options.Label)} property, not both."); - } - - if (options.FeatureFlagSelectors.Count() == 0) - { - // Select clause is not present - options.FeatureFlagSelectors.Add(new KeyValueSelector - { - KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, - IsFeatureFlagSelector = true - }); - } - - foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) - { - _selectors.AppendUnique(featureFlagSelector); - - _ffWatchers.AppendUnique(new KeyValueWatcher - { - Key = featureFlagSelector.KeyFilter, - Label = featureFlagSelector.LabelFilter, - // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins - RefreshInterval = options.RefreshInterval - }); - } - - return this; - } - - /// - /// Connect the provider to the Azure App Configuration service via a connection string. - /// - /// - /// Used to authenticate with Azure App Configuration. - /// - public AzureAppConfigurationOptions Connect(string connectionString) - { - if (string.IsNullOrWhiteSpace(connectionString)) - { - throw new ArgumentNullException(nameof(connectionString)); - } - - return Connect(new List { connectionString }); - } - - /// - /// Connect the provider to an Azure App Configuration store and its replicas via a list of connection strings. - /// - /// - /// Used to authenticate with Azure App Configuration. - /// - public AzureAppConfigurationOptions Connect(IEnumerable connectionStrings) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - if (connectionStrings.Distinct().Count() != connectionStrings.Count()) - { - throw new ArgumentException($"All values in '{nameof(connectionStrings)}' must be unique."); - } - - Endpoints = null; - Credential = null; - ConnectionStrings = connectionStrings; - return this; - } - - /// - /// Connect the provider to Azure App Configuration using endpoint and token credentials. - /// - /// The endpoint of the Azure App Configuration to connect to. - /// Token credentials to use to connect. - public AzureAppConfigurationOptions Connect(Uri endpoint, TokenCredential credential) - { - if (endpoint == null) - { - throw new ArgumentNullException(nameof(endpoint)); - } - - if (credential == null) - { - throw new ArgumentNullException(nameof(credential)); - } - - return Connect(new List() { endpoint }, credential); - } - - /// - /// Connect the provider to an Azure App Configuration store and its replicas using a list of endpoints and a token credential. - /// - /// The list of endpoints of an Azure App Configuration store and its replicas to connect to. - /// Token credential to use to connect. - public AzureAppConfigurationOptions Connect(IEnumerable endpoints, TokenCredential credential) - { - if (endpoints == null || !endpoints.Any()) - { - throw new ArgumentNullException(nameof(endpoints)); - } - - if (endpoints.Distinct(new EndpointComparer()).Count() != endpoints.Count()) - { - throw new ArgumentException($"All values in '{nameof(endpoints)}' must be unique."); - } - - Credential = credential ?? throw new ArgumentNullException(nameof(credential)); - - Endpoints = endpoints; - ConnectionStrings = null; - return this; - } - - /// - /// Trims the provided prefix from the keys of all key-values retrieved from Azure App Configuration. - /// - /// The prefix to be trimmed. - public AzureAppConfigurationOptions TrimKeyPrefix(string prefix) - { - if (string.IsNullOrEmpty(prefix)) - { - throw new ArgumentNullException(nameof(prefix)); - } - - _keyPrefixes.Add(prefix); - return this; - } - - /// - /// Configure the client(s) used to communicate with Azure App Configuration. - /// - /// A callback used to configure Azure App Configuration client options. - public AzureAppConfigurationOptions ConfigureClientOptions(Action configure) - { - configure?.Invoke(ClientOptions); - return this; - } - - /// - /// Configure refresh for key-values in the configuration provider. - /// - /// A callback used to configure Azure App Configuration refresh options. - public AzureAppConfigurationOptions ConfigureRefresh(Action configure) - { - if (RegisterAllEnabled) - { - throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); - } - - var refreshOptions = new AzureAppConfigurationRefreshOptions(); - configure?.Invoke(refreshOptions); - - bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); - RegisterAllEnabled = refreshOptions.RegisterAllEnabled; - - if (!isRegisterCalled && !RegisterAllEnabled) - { - throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + - $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); - } - - // Check if both register methods are called at any point - if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) - { - throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " - + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); - } - - if (RegisterAllEnabled) - { - KvCollectionRefreshInterval = refreshOptions.RefreshInterval; - } - else - { - foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) - { - item.RefreshInterval = refreshOptions.RefreshInterval; - _individualKvWatchers.Add(item); - } - } - - return this; - } - - /// - /// Get an instance of that can be used to trigger a refresh for the registered key-values. - /// - /// An instance of . - public IConfigurationRefresher GetRefresher() - { - return _refresher; - } - - /// - /// Configures the Azure App Configuration provider to use the provided Key Vault configuration to resolve key vault references. - /// - /// A callback used to configure Azure App Configuration key vault options. - public AzureAppConfigurationOptions ConfigureKeyVault(Action configure) - { - var keyVaultOptions = new AzureAppConfigurationKeyVaultOptions(); - configure?.Invoke(keyVaultOptions); - - if (keyVaultOptions.Credential != null && keyVaultOptions.SecretResolver != null) - { - throw new InvalidOperationException($"Cannot configure both default credentials and secret resolver for Key Vault references. Please call either {nameof(keyVaultOptions.SetCredential)} or {nameof(keyVaultOptions.SetSecretResolver)} method, not both."); - } - - _adapters.RemoveAll(a => a is AzureKeyVaultKeyValueAdapter); - _adapters.Add(new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider(keyVaultOptions))); - - IsKeyVaultRefreshConfigured = keyVaultOptions.IsKeyVaultRefreshConfigured; - IsKeyVaultConfigured = true; - return this; - } - - /// - /// Provides a way to transform settings retrieved from App Configuration before they are processed by the configuration provider. - /// - /// A callback registered by the user to transform each configuration setting. - public AzureAppConfigurationOptions Map(Func> mapper) - { - if (mapper == null) - { - throw new ArgumentNullException(nameof(mapper)); - } - - _mappers.Add(mapper); - return this; - } - - /// - /// Configure the provider behavior when loading data from Azure App Configuration on startup. - /// - /// A callback used to configure Azure App Configuration startup options. - public AzureAppConfigurationOptions ConfigureStartupOptions(Action configure) - { - configure?.Invoke(Startup); - return this; - } - - private static ConfigurationClientOptions GetDefaultClientOptions() - { - var clientOptions = new ConfigurationClientOptions(ConfigurationClientOptions.ServiceVersion.V2023_10_01); - clientOptions.Retry.MaxRetries = MaxRetries; - clientOptions.Retry.MaxDelay = MaxRetryDelay; - clientOptions.Retry.Mode = RetryMode.Exponential; - clientOptions.AddPolicy(new UserAgentHeaderPolicy(), HttpPipelinePosition.PerCall); - - return clientOptions; - } - } -} + + return clientOptions; + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 9d61faca..795aeb10 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -1209,6 +1209,13 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => private bool IsFailOverable(AggregateException ex) { + TaskCanceledException tce = ex.InnerExceptions?.LastOrDefault(e => e is TaskCanceledException) as TaskCanceledException; + + if (tce != null && tce.InnerException is TimeoutException) + { + return true; + } + RequestFailedException rfe = ex.InnerExceptions?.LastOrDefault(e => e is RequestFailedException) as RequestFailedException; return rfe != null ? IsFailOverable(rfe) : false; From d7f5939c7fe1038c558b10561ea61af06ed768fb Mon Sep 17 00:00:00 2001 From: Sami Sadfa <71456174+samsadsam@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:18:35 -0400 Subject: [PATCH 22/22] upgrade to 8.2.0-preview (#638) --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index a5d4dee4..fda406b9 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 4354b77e..eb43831e 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 369f5dbd..dc558788 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -36,7 +36,7 @@ - 8.1.0-preview + 8.2.0-preview