diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index a5d4dee4..fda406b9 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 4354b77e..eb43831e 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs new file mode 100644 index 00000000..6127822d --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationClientFactory.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Azure.Core; +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal class AzureAppConfigurationClientFactory : IAzureClientFactory + { + private readonly ConfigurationClientOptions _clientOptions; + + private readonly TokenCredential _credential; + private readonly IEnumerable _connectionStrings; + + public AzureAppConfigurationClientFactory( + IEnumerable connectionStrings, + ConfigurationClientOptions clientOptions) + { + if (connectionStrings == null || !connectionStrings.Any()) + { + throw new ArgumentNullException(nameof(connectionStrings)); + } + + _connectionStrings = connectionStrings; + + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public AzureAppConfigurationClientFactory( + TokenCredential credential, + ConfigurationClientOptions clientOptions) + { + _credential = credential ?? throw new ArgumentNullException(nameof(credential)); + _clientOptions = clientOptions ?? throw new ArgumentNullException(nameof(clientOptions)); + } + + public ConfigurationClient CreateClient(string endpoint) + { + if (string.IsNullOrEmpty(endpoint)) + { + throw new ArgumentNullException(nameof(endpoint)); + } + + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out Uri uriResult)) + { + throw new ArgumentException("Invalid host URI."); + } + + if (_credential != null) + { + return new ConfigurationClient(uriResult, _credential, _clientOptions); + } + + string connectionString = _connectionStrings.FirstOrDefault(cs => ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection) == endpoint); + + // + // falback to the first connection string + if (connectionString == null) + { + string id = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.IdSection); + string secret = ConnectionStringUtils.Parse(_connectionStrings.First(), ConnectionStringUtils.SecretSection); + + connectionString = ConnectionStringUtils.Build(uriResult, id, secret); + } + + return new ConfigurationClient(connectionString, _clientOptions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs index dc022bac..8b4bf8c8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs @@ -33,7 +33,9 @@ private static bool IsProviderDisabled() /// /// The configuration builder to add key-values to. /// The connection string used to connect to the configuration store. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, @@ -48,7 +50,9 @@ public static IConfigurationBuilder AddAzureAppConfiguration( /// /// The configuration builder to add key-values to. /// The list of connection strings used to connect to the configuration store and its replicas. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, @@ -63,7 +67,9 @@ public static IConfigurationBuilder AddAzureAppConfiguration( /// /// The configuration builder to add key-values to. /// A callback used to configure Azure App Configuration options. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7d9a9cad..804d5d88 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -2,7 +2,9 @@ // Licensed under the MIT license. // using Azure.Core; +using Azure.Core.Pipeline; using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; @@ -10,6 +12,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration @@ -22,13 +25,16 @@ public class AzureAppConfigurationOptions { private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); + private static readonly TimeSpan NetworkTimeout = TimeSpan.FromSeconds(10); + private static readonly KeyValueSelector DefaultQuery = new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null }; - private List _changeWatchers = new List(); - private List _multiKeyWatchers = new List(); + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors = new List(); + private List _selectors; private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; // The following set is sorted in descending order. // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. @@ -62,19 +68,29 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// A collection of . + /// A collection of specified by user. /// - internal IEnumerable KeyValueSelectors => _kvSelectors; + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } /// /// A collection of . /// - internal IEnumerable ChangeWatchers => _changeWatchers; + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable FeatureFlagWatchers => _ffWatchers; /// /// A collection of . @@ -96,11 +112,15 @@ internal IEnumerable Adapters internal IEnumerable KeyPrefixes => _keyPrefixes; /// - /// An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. /// - /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. /// @@ -131,6 +151,11 @@ internal IEnumerable Adapters /// internal StartupOptions Startup { get; set; } = new StartupOptions(); + /// + /// Client factory that is responsible for creating instances of ConfigurationClient. + /// + internal IAzureClientFactory ClientFactory { get; private set; } + /// /// Initializes a new instance of the class. /// @@ -142,6 +167,20 @@ public AzureAppConfigurationOptions() new JsonKeyValueAdapter(), new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { DefaultQuery }; + } + + /// + /// Sets the client factory used to create ConfigurationClient instances. + /// + /// The client factory. + /// The current instance. + public AzureAppConfigurationOptions SetClientFactory(IAzureClientFactory factory) + { + ClientFactory = factory ?? throw new ArgumentNullException(nameof(factory)); + return this; } /// @@ -170,22 +209,30 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter throw new ArgumentNullException(nameof(keyFilter)); } + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + if (string.IsNullOrWhiteSpace(labelFilter)) { labelFilter = LabelFilter.Null; } - // Do not support * and , for label filter for now. - if (labelFilter.Contains('*') || labelFilter.Contains(',')) + if (!_selectCalled) { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + _selectors.Remove(DefaultQuery); + + _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { KeyFilter = keyFilter, LabelFilter = labelFilter }); + return this; } @@ -201,7 +248,14 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) throw new ArgumentNullException(nameof(name)); } - _kvSelectors.AppendUnique(new KeyValueSelector + if (!_selectCalled) + { + _selectors.Remove(DefaultQuery); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector { SnapshotName = name }); @@ -212,7 +266,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh on an individual flag level. + /// All loaded feature flags will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) @@ -237,25 +291,22 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = options.Label == null ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true }); } - foreach (var featureFlagSelector in options.FeatureFlagSelectors) + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - var featureFlagFilter = featureFlagSelector.KeyFilter; - var labelFilter = featureFlagSelector.LabelFilter; + _selectors.AppendUnique(featureFlagSelector); - Select(featureFlagFilter, labelFilter); - - _multiKeyWatchers.AppendUnique(new KeyValueWatcher + _ffWatchers.AppendUnique(new KeyValueWatcher { - Key = featureFlagFilter, - Label = labelFilter, + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); - } return this; @@ -376,18 +427,41 @@ public AzureAppConfigurationOptions ConfigureClientOptions(ActionA callback used to configure Azure App Configuration refresh options. public AzureAppConfigurationOptions ConfigureRefresh(Action configure) { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + var refreshOptions = new AzureAppConfigurationRefreshOptions(); configure?.Invoke(refreshOptions); - if (!refreshOptions.RefreshRegistrations.Any()) + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) { - throw new ArgumentException($"{nameof(ConfigureRefresh)}() must have at least one key-value registered for refresh."); + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); } - foreach (var item in refreshOptions.RefreshRegistrations) + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) { - item.RefreshInterval = refreshOptions.RefreshInterval; - _changeWatchers.Add(item); + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); + } + + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else + { + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } } return this; @@ -456,6 +530,10 @@ private static ConfigurationClientOptions GetDefaultClientOptions() clientOptions.Retry.MaxDelay = MaxRetryDelay; clientOptions.Retry.Mode = RetryMode.Exponential; clientOptions.AddPolicy(new UserAgentHeaderPolicy(), HttpPipelinePosition.PerCall); + clientOptions.Transport = new HttpClientTransport(new HttpClient() + { + Timeout = NetworkTimeout + }); return clientOptions; } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b5dd42f1..795aeb10 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -4,7 +4,6 @@ using Azure; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using Microsoft.Extensions.Logging; using System; @@ -32,9 +31,13 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); + private Dictionary _watchedIndividualKvs = new Dictionary(); + private HashSet _ffKeys = new HashSet(); + private Dictionary> _kvEtags = new Dictionary>(); + private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); + private DateTimeOffset _nextCollectionRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -108,11 +111,25 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); + IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (watchers.Any()) + bool hasWatchers = watchers.Any(); + TimeSpan minWatcherRefreshInterval = hasWatchers ? watchers.Min(w => w.RefreshInterval) : TimeSpan.MaxValue; + + if (options.RegisterAllEnabled) + { + if (options.KvCollectionRefreshInterval <= TimeSpan.Zero) + { + throw new ArgumentException( + $"{nameof(options.KvCollectionRefreshInterval)} must be greater than zero seconds when using RegisterAll for refresh", + nameof(options)); + } + + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); + } + else if (hasWatchers) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + MinRefreshInterval = minWatcherRefreshInterval; } else { @@ -194,13 +211,15 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableMultiKeyWatchers = _options.MultiKeyWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); + bool isRefreshDue = _options.RegisterAllEnabled && utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableWatchers.Any() && - !refreshableMultiKeyWatchers.Any() && + !refreshableIndividualKvWatchers.Any() && + !refreshableFfWatchers.Any() && + !isRefreshDue && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -208,6 +227,11 @@ public async Task RefreshAsync(CancellationToken cancellationToken) IEnumerable clients = _configClientManager.GetClients(); + if (_requestTracingOptions != null) + { + _requestTracingOptions.ReplicaCount = clients.Count() - 1; + } + // // Filter clients based on their backoff status clients = clients.Where(client => @@ -249,179 +273,166 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary watchedSettings = null; + Dictionary> kvEtags = null; + Dictionary> ffEtags = null; + HashSet ffKeys = null; + Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - List changedKeyValuesCollection = null; Dictionary data = null; + Dictionary ffCollectionData = null; + bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => - { - data = null; - watchedSettings = null; - keyValueChanges = new List(); - changedKeyValuesCollection = null; - refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); - logDebugBuilder.Clear(); - logInfoBuilder.Clear(); - - foreach (KeyValueWatcher changeWatcher in refreshableWatchers) - { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; - - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); - - KeyValueChange change = default; - - // - // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) - { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - else - { - // Load the key-value in case the previous load attempts had failed - - try - { - await CallWithRequestTracing( - async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) - { - watchedKv = null; - } - - if (watchedKv != null) - { - change = new KeyValueChange() - { - Key = watchedKv.Key, - Label = watchedKv.Label.NormalizeNull(), - Current = watchedKv, - ChangeType = KeyValueChangeType.Modified - }; - } - } - - // Check if a change has been detected in the key-value registered for refresh - if (change.ChangeType != KeyValueChangeType.None) - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); - keyValueChanges.Add(change); - - if (changeWatcher.RefreshAll) - { - refreshAll = true; - break; - } - } - else - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - } - } + { + kvEtags = null; + ffEtags = null; + ffKeys = null; + watchedIndividualKvs = null; + keyValueChanges = new List(); + data = null; + ffCollectionData = null; + ffCollectionUpdated = false; + refreshAll = false; + logDebugBuilder.Clear(); + logInfoBuilder.Clear(); + Uri endpoint = _configClientManager.GetEndpointForClient(client); - if (refreshAll) + if (_options.RegisterAllEnabled) + { + // Get key value collection changes if RegisterAll was called + if (isRefreshDue) { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; + refreshAll = await HaveCollectionsChanged( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector), + _kvEtags, + client, + cancellationToken).ConfigureAwait(false); } + } + else + { + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); + } - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + if (refreshAll) + { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true, + // or if any key-value collection change was detected. + kvEtags = new Dictionary>(); + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, ffKeys, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } - if (!changedKeyValuesCollection.Any()) + // Get feature flag changes + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - }, - cancellationToken) - .ConfigureAwait(false); + KeyFilter = watcher.Key, + LabelFilter = watcher.Label, + IsFeatureFlagSelector = true + }), + _ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) + { + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + ffKeys, + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + }, + cancellationToken) + .ConfigureAwait(false); - if (!refreshAll) + if (refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) + // Invalidate all the cached KeyVault secrets + foreach (IKeyValueAdapter adapter in _options.Adapters) { - UpdateNextRefreshTime(changeWatcher); + adapter.OnChangeDetected(); } - foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); - if (change.ChangeType == KeyValueChangeType.Modified) - { - ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + UpdateNextRefreshTime(changeWatcher); + } + } + else + { + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - foreach (Func> func in _options.Mappers) - { - setting = await func(setting).ConfigureAwait(false); - } + await ProcessKeyValueChangesAsync(keyValueChanges, _mappedData, watchedIndividualKvs).ConfigureAwait(false); - if (setting == null) - { - _mappedData.Remove(change.Key); - } - else - { - _mappedData[change.Key] = setting; - } - } - else if (change.ChangeType == KeyValueChangeType.Deleted) + if (ffCollectionUpdated) + { + // Remove all feature flag keys that are not present in the latest loading of feature flags, but were loaded previously + foreach (string key in _ffKeys.Except(ffKeys)) { - _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + _mappedData.Remove(key); } - // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting - foreach (IKeyValueAdapter adapter in _options.Adapters) + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + foreach (KeyValuePair kvp in mappedFfData) { - // If the current setting is null, try to pass the previous setting instead - if (change.Current != null) - { - adapter.OnChangeDetected(change.Current); - } - else if (change.Previous != null) - { - adapter.OnChangeDetected(change.Previous); - } + _mappedData[kvp.Key] = kvp.Value; } } - } - else - { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - - // Invalidate all the cached KeyVault secrets - foreach (IKeyValueAdapter adapter in _options.Adapters) - { - adapter.OnChangeDetected(); - } - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + // + // update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { UpdateNextRefreshTime(changeWatcher); } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || changedKeyValuesCollection?.Any() == true || keyValueChanges.Any()) + if (_options.RegisterAllEnabled && isRefreshDue) + { + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) { - _watchedSettings = watchedSettings; + _watchedIndividualKvs = watchedIndividualKvs ?? _watchedIndividualKvs; + + _ffEtags = ffEtags ?? _ffEtags; + + _kvEtags = kvEtags ?? _kvEtags; + + _ffKeys = ffKeys ?? _ffKeys; if (logDebugBuilder.Length > 0) { @@ -432,6 +443,7 @@ await CallWithRequestTracing( { _logger.LogInformation(logInfoBuilder.ToString().Trim()); } + // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. @@ -543,6 +555,11 @@ public void ProcessPushNotification(PushNotification pushNotification, TimeSpan? if (_configClientManager.UpdateSyncToken(pushNotification.ResourceUri, pushNotification.SyncToken)) { + if (_requestTracingEnabled && _requestTracingOptions != null) + { + _requestTracingOptions.IsPushRefreshUsed = true; + } + SetDirty(maxDelay); } else @@ -555,14 +572,21 @@ private void SetDirty(TimeSpan? maxDelay) { DateTimeOffset nextRefreshTime = AddRandomDelay(DateTimeOffset.UtcNow, maxDelay ?? DefaultMaxSetDirtyDelay); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + if (_options.RegisterAllEnabled) { - changeWatcher.NextRefreshTime = nextRefreshTime; + _nextCollectionRefreshTime = nextRefreshTime; + } + else + { + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) + { + kvWatcher.NextRefreshTime = nextRefreshTime; + } } - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { - changeWatcher.NextRefreshTime = nextRefreshTime; + featureFlagWatcher.NextRefreshTime = nextRefreshTime; } } @@ -612,6 +636,11 @@ private async Task LoadAsync(bool ignoreFailures, CancellationToken cancellation { IEnumerable clients = _configClientManager.GetClients(); + if (_requestTracingOptions != null) + { + _requestTracingOptions.ReplicaCount = clients.Count() - 1; + } + if (await TryInitializeAsync(clients, startupExceptions, cancellationToken).ConfigureAwait(false)) { break; @@ -707,34 +736,44 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary watchedSettings = null; + Dictionary> kvEtags = new Dictionary>(); + Dictionary> ffEtags = new Dictionary>(); + Dictionary watchedIndividualKvs = null; + HashSet ffKeys = new HashSet(); await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - data = await LoadSelectedKeyValues( + data = await LoadSelected( client, + kvEtags, + ffEtags, + _options.Selectors, + ffKeys, cancellationToken) .ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh( + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) .ConfigureAwait(false); - - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); }, cancellationToken) .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + if (_options.RegisterAllEnabled) + { + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + if (data != null) { // Invalidate all the cached KeyVault secrets @@ -744,51 +783,71 @@ await ExecuteWithFailOverPolicyAsync( } Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; + _mappedData = mappedData; + _kvEtags = kvEtags; + _ffEtags = ffEtags; + _watchedIndividualKvs = watchedIndividualKvs; + _ffKeys = ffKeys; } } - private async Task> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task> LoadSelected( + ConfigurationClient client, + Dictionary> kvEtags, + Dictionary> ffEtags, + IEnumerable selectors, + HashSet ffKeys, + CancellationToken cancellationToken) { - var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); - - // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || - !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); + Dictionary data = new Dictionary(); - if (useDefaultQuery) + foreach (KeyValueSelector loadOption in selectors) { - // Load all key-values with the null label. - var selector = new SettingSelector - { - KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null - }; - - await CallWithRequestTracing(async () => + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + var selector = new SettingSelector() { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); - } + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }; - foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) - { - IAsyncEnumerable settingsEnumerable; + var matchConditions = new List(); - if (string.IsNullOrEmpty(loadOption.SnapshotName)) - { - settingsEnumerable = client.GetConfigurationSettingsAsync( - new SettingSelector + await CallWithRequestTracing(async () => + { + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages(_options.ConfigurationSettingPageIterator).ConfigureAwait(false)) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }, - cancellationToken); + using Response response = page.GetRawResponse(); + + foreach (ConfigurationSetting setting in page.Values) + { + data[setting.Key] = setting; + + if (loadOption.IsFeatureFlagSelector) + { + ffKeys.Add(setting.Key); + } + } + + // The ETag will never be null here because it's not a conditional request + // Each successful response should have 200 status code and an ETag + matchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); + } + }).ConfigureAwait(false); + + if (loadOption.IsFeatureFlagSelector) + { + ffEtags[loadOption] = matchConditions; + } + else + { + kvEtags[loadOption] = matchConditions; + } } else { @@ -808,38 +867,42 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + IAsyncEnumerable settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( loadOption.SnapshotName, cancellationToken); - } - await CallWithRequestTracing(async () => - { - await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + await CallWithRequestTracing(async () => { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + { + data[setting.Key] = setting; + } + }).ConfigureAwait(false); + } } - return serverData; + return data; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh( + ConfigurationClient client, + IDictionary existingSettings, + CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + var watchedIndividualKvs = new Dictionary(); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -857,61 +920,84 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSettings; + return watchedIndividualKvs; } - private Dictionary UpdateWatchedKeyValueCollections(Dictionary watchedSettings, IDictionary existingSettings) + private async Task RefreshIndividualKvWatchers( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableIndividualKvWatchers, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher kvWatcher in refreshableIndividualKvWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + KeyValueChange change = default; - foreach (ConfigurationSetting setting in currentKeyValues) + // + // Find if there is a change associated with watcher + if (_watchedIndividualKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - } + else + { + // Load the key-value in case the previous load attempts had failed - return watchedSettings; - } + try + { + await CallWithRequestTracing( + async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) + { + watchedKv = null; + } - private async Task> GetRefreshedKeyValueCollections( - IEnumerable multiKeyWatchers, - ConfigurationClient client, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) - { - var keyValueChanges = new List(); + if (watchedKv != null) + { + change = new KeyValueChange() + { + Key = watchedKv.Key, + Label = watchedKv.Label.NormalizeNull(), + Current = watchedKv, + ChangeType = KeyValueChangeType.Modified + }; + } + } - foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) - { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, _watchedSettings.Values); + // Check if a change has been detected in the key-value registered for refresh + if (change.ChangeType != KeyValueChangeType.None) + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); + keyValueChanges.Add(change); - keyValueChanges.AddRange( - await client.GetKeyValueChangeCollection( - currentKeyValues, - new GetKeyValueChangeCollectionOptions - { - KeyFilter = changeWatcher.Key, - Label = changeWatcher.Label.NormalizeNull(), - RequestTracingEnabled = _requestTracingEnabled, - RequestTracingOptions = _requestTracingOptions - }, - logDebugBuilder, - logInfoBuilder, - endpoint, - cancellationToken) - .ConfigureAwait(false)); + if (kvWatcher.RefreshAll) + { + return true; + } + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + } } - return keyValueChanges; + return false; } private void SetData(IDictionary data) @@ -966,7 +1052,6 @@ private void SetRequestTracingOptions() IsDevEnvironment = TracingUtils.IsDevEnvironment(), IsKeyVaultConfigured = _options.IsKeyVaultConfigured, IsKeyVaultRefreshConfigured = _options.IsKeyVaultRefreshConfigured, - ReplicaCount = _options.Endpoints?.Count() - 1 ?? _options.ConnectionStrings?.Count() - 1 ?? 0, FeatureFlagTracing = _options.FeatureFlagTracing, IsLoadBalancingEnabled = _options.LoadBalancingEnabled }; @@ -1124,6 +1209,13 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => private bool IsFailOverable(AggregateException ex) { + TaskCanceledException tce = ex.InnerExceptions?.LastOrDefault(e => e is TaskCanceledException) as TaskCanceledException; + + if (tce != null && tce.InnerException is TimeoutException) + { + return true; + } + RequestFailedException rfe = ex.InnerExceptions?.LastOrDefault(e => e is RequestFailedException) as RequestFailedException; return rfe != null ? IsFailOverable(rfe) : false; @@ -1179,30 +1271,6 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingSettings) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.StartsWith(keyPrefix) && kv.Label == label.NormalizeNull(); - }); - } - else - { - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.Equals(key) && kv.Label == label.NormalizeNull(); - }); - } - - return currentKeyValues; - } - private void EnsureAssemblyInspected() { if (!_isAssemblyInspected) @@ -1248,6 +1316,88 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } + private async Task HaveCollectionsChanged( + IEnumerable selectors, + Dictionary> pageEtags, + ConfigurationClient client, + CancellationToken cancellationToken) + { + bool haveCollectionsChanged = false; + + foreach (KeyValueSelector selector in selectors) + { + if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => haveCollectionsChanged = await client.HaveCollectionsChanged( + selector, + matchConditions, + _options.ConfigurationSettingPageIterator, + cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (haveCollectionsChanged) + { + return true; + } + } + + return haveCollectionsChanged; + } + + private async Task ProcessKeyValueChangesAsync( + IEnumerable keyValueChanges, + Dictionary mappedData, + Dictionary watchedIndividualKvs) + { + foreach (KeyValueChange change in keyValueChanges) + { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + + if (change.ChangeType == KeyValueChangeType.Modified) + { + ConfigurationSetting setting = change.Current; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + watchedIndividualKvs[changeIdentifier] = settingCopy; + + foreach (Func> func in _options.Mappers) + { + setting = await func(setting).ConfigureAwait(false); + } + + if (setting == null) + { + mappedData.Remove(change.Key); + } + else + { + mappedData[change.Key] = setting; + } + } + else if (change.ChangeType == KeyValueChangeType.Deleted) + { + mappedData.Remove(change.Key); + + watchedIndividualKvs.Remove(changeIdentifier); + } + + // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting + foreach (IKeyValueAdapter adapter in _options.Adapters) + { + // If the current setting is null, try to pass the previous setting instead + if (change.Current != null) + { + adapter.OnChangeDetected(change.Current); + } + else if (change.Previous != null) + { + adapter.OnChangeDetected(change.Previous); + } + } + } + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index f3fb6c4a..cf2847a8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -14,6 +14,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); + internal bool RegisterAllEnabled { get; private set; } /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. @@ -50,6 +51,17 @@ public AzureAppConfigurationRefreshOptions Register(string key, string label = L return this; } + /// + /// Register all key-values loaded outside of to be refreshed when the configuration provider's triggers a refresh. + /// The instance can be obtained by calling . + /// + public AzureAppConfigurationRefreshOptions RegisterAll() + { + RegisterAllEnabled = true; + + return this; + } + /// /// Sets the cache expiration time for the key-values registered for refresh. Default value is 30 seconds. Must be greater than 1 second. /// Any refresh operation triggered using will not update the value for a key until the cached value for that key has expired. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs index dee62006..83d20e2f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationSource.cs @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Azure; using System; +using System.Collections.Generic; +using System.Linq; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -29,35 +33,33 @@ public IConfigurationProvider Build(IConfigurationBuilder builder) try { AzureAppConfigurationOptions options = _optionsProvider(); - IConfigurationClientManager clientManager; if (options.ClientManager != null) { - clientManager = options.ClientManager; + return new AzureAppConfigurationProvider(options.ClientManager, options, _optional); } - else if (options.ConnectionStrings != null) + + IEnumerable endpoints; + IAzureClientFactory clientFactory = options.ClientFactory; + + if (options.ConnectionStrings != null) { - clientManager = new ConfigurationClientManager( - options.ConnectionStrings, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.ConnectionStrings.Select(cs => new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection))); + + clientFactory ??= new AzureAppConfigurationClientFactory(options.ConnectionStrings, options.ClientOptions); } else if (options.Endpoints != null && options.Credential != null) { - clientManager = new ConfigurationClientManager( - options.Endpoints, - options.Credential, - options.ClientOptions, - options.ReplicaDiscoveryEnabled, - options.LoadBalancingEnabled); + endpoints = options.Endpoints; + + clientFactory ??= new AzureAppConfigurationClientFactory(options.Credential, options.ClientOptions); } else { throw new ArgumentException($"Please call {nameof(AzureAppConfigurationOptions)}.{nameof(AzureAppConfigurationOptions.Connect)} to specify how to connect to Azure App Configuration."); } - provider = new AzureAppConfigurationProvider(clientManager, options, _optional); + provider = new AzureAppConfigurationProvider(new ConfigurationClientManager(clientFactory, endpoints, options.ReplicaDiscoveryEnabled, options.LoadBalancingEnabled), options, _optional); } catch (InvalidOperationException ex) // InvalidOperationException is thrown when any problems are found while configuring AzureAppConfigurationOptions or when SDK fails to create a configurationClient. { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs index a0215ca3..61840d03 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs @@ -2,10 +2,10 @@ // Licensed under the MIT license. // -using Azure.Core; using Azure.Data.AppConfiguration; using DnsClient; using DnsClient.Protocol; +using Microsoft.Extensions.Azure; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; @@ -26,12 +26,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration /// internal class ConfigurationClientManager : IConfigurationClientManager, IDisposable { + private readonly IAzureClientFactory _clientFactory; private readonly IList _clients; + private readonly Uri _endpoint; - private readonly string _secret; - private readonly string _id; - private readonly TokenCredential _credential; - private readonly ConfigurationClientOptions _clientOptions; + private readonly bool _replicaDiscoveryEnabled; private readonly SrvLookupClient _srvLookupClient; private readonly string _validDomain; @@ -52,61 +51,20 @@ internal class ConfigurationClientManager : IConfigurationClientManager, IDispos internal int RefreshClientsCalled { get; set; } = 0; public ConfigurationClientManager( - IEnumerable connectionStrings, - ConfigurationClientOptions clientOptions, - bool replicaDiscoveryEnabled, - bool loadBalancingEnabled) - { - if (connectionStrings == null || !connectionStrings.Any()) - { - throw new ArgumentNullException(nameof(connectionStrings)); - } - - string connectionString = connectionStrings.First(); - _endpoint = new Uri(ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.EndpointSection)); - _secret = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.SecretSection); - _id = ConnectionStringUtils.Parse(connectionString, ConnectionStringUtils.IdSection); - _clientOptions = clientOptions; - _replicaDiscoveryEnabled = replicaDiscoveryEnabled; - - // If load balancing is enabled, shuffle the passed in connection strings to randomize the endpoint used on startup - if (loadBalancingEnabled) - { - connectionStrings = connectionStrings.ToList().Shuffle(); - } - - _validDomain = GetValidDomain(_endpoint); - _srvLookupClient = new SrvLookupClient(); - - _clients = connectionStrings - .Select(cs => - { - var endpoint = new Uri(ConnectionStringUtils.Parse(cs, ConnectionStringUtils.EndpointSection)); - return new ConfigurationClientWrapper(endpoint, new ConfigurationClient(cs, _clientOptions)); - }) - .ToList(); - } - - public ConfigurationClientManager( + IAzureClientFactory clientFactory, IEnumerable endpoints, - TokenCredential credential, - ConfigurationClientOptions clientOptions, bool replicaDiscoveryEnabled, bool loadBalancingEnabled) { + _clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory)); + if (endpoints == null || !endpoints.Any()) { throw new ArgumentNullException(nameof(endpoints)); } - if (credential == null) - { - throw new ArgumentNullException(nameof(credential)); - } - _endpoint = endpoints.First(); - _credential = credential; - _clientOptions = clientOptions; + _replicaDiscoveryEnabled = replicaDiscoveryEnabled; // If load balancing is enabled, shuffle the passed in endpoints to randomize the endpoint used on startup @@ -119,7 +77,7 @@ public ConfigurationClientManager( _srvLookupClient = new SrvLookupClient(); _clients = endpoints - .Select(endpoint => new ConfigurationClientWrapper(endpoint, new ConfigurationClient(endpoint, _credential, _clientOptions))) + .Select(endpoint => new ConfigurationClientWrapper(endpoint, clientFactory.CreateClient(endpoint.AbsoluteUri))) .ToList(); } @@ -289,9 +247,7 @@ private async Task RefreshFallbackClients(CancellationToken cancellationToken) { var targetEndpoint = new Uri($"https://{host}"); - var configClient = _credential == null - ? new ConfigurationClient(ConnectionStringUtils.Build(targetEndpoint, _id, _secret), _clientOptions) - : new ConfigurationClient(targetEndpoint, _credential, _clientOptions); + ConfigurationClient configClient = _clientFactory.CreateClient(targetEndpoint.AbsoluteUri); newDynamicClients.Add(new ConfigurationClientWrapper(targetEndpoint, configClient)); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs new file mode 100644 index 00000000..aba7684b --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs @@ -0,0 +1,33 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs new file mode 100644 index 00000000..af7f1f03 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + /// + /// Provides extension methods for configuration sources. + /// + public static class ConfigurationSourceExtensions + { + /// + /// Determines whether the specified configuration source is an Azure App Configuration source. + /// + /// The configuration source to check. + /// true if the specified source is an Azure App Configuration source; otherwise, false. + public static bool IsAzureAppConfigurationSource(this IConfigurationSource source) + { + return source is AzureAppConfigurationSource; + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 86576a48..3bdcaecd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -20,14 +20,15 @@ internal class LoggingConstants // Successful update, debug log level public const string RefreshKeyValueRead = "Key-value read from App Configuration."; public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; - public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValueCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; - public const string RefreshFeatureFlagUpdated = "Feature flag updated."; + public const string RefreshFeatureFlagsUpdated = "Feature flags reloaded."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags reloaded."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs index 15e862b6..f732ab95 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs @@ -33,6 +33,7 @@ internal class RequestTracingConstants public const string LoadBalancingEnabledTag = "LB"; public const string SignalRUsedTag = "SignalR"; public const string FailoverRequestTag = "Failover"; + public const string PushRefreshTag = "PushRefresh"; public const string FeatureFlagFilterTypeKey = "Filter"; public const string CustomFilter = "CSTM"; @@ -43,7 +44,6 @@ internal class RequestTracingConstants public const string FeatureFlagUsesTelemetryTag = "Telemetry"; public const string FeatureFlagUsesSeedTag = "Seed"; public const string FeatureFlagMaxVariantsKey = "MaxVariants"; - public const string FeatureFlagUsesVariantConfigurationReferenceTag = "ConfigRef"; public const string DiagnosticHeaderActivityName = "Azure.CustomDiagnosticHeaders"; public const string CorrelationContextHeader = "Correlation-Context"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index d479ad6b..c4edfb0e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,12 +3,10 @@ // using Azure; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -31,7 +29,8 @@ public static async Task GetKeyValueChange(this ConfigurationCli try { Response response = await client.GetConfigurationSettingAsync(setting, onlyIfChanged: true, cancellationToken).ConfigureAwait(false); - if (response.GetRawResponse().Status == (int)HttpStatusCode.OK) + if (response.GetRawResponse().Status == (int)HttpStatusCode.OK && + !response.Value.ETag.Equals(setting.ETag)) { return new KeyValueChange { @@ -65,131 +64,48 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetKeyValueChangeCollection( - this ConfigurationClient client, - IEnumerable keyValues, - GetKeyValueChangeCollectionOptions options, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IConfigurationSettingPageIterator pageIterator, CancellationToken cancellationToken) { - if (options == null) + if (matchConditions == null) { - throw new ArgumentNullException(nameof(options)); + throw new ArgumentNullException(nameof(matchConditions)); } - if (keyValues == null) + if (keyValueSelector == null) { - keyValues = Enumerable.Empty(); + throw new ArgumentNullException(nameof(keyValueSelector)); } - if (options.KeyFilter == null) + if (keyValueSelector.SnapshotName != null) { - options.KeyFilter = string.Empty; + throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); } - if (keyValues.Any(k => string.IsNullOrEmpty(k.Key))) + SettingSelector selector = new SettingSelector { - throw new ArgumentNullException($"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Key)}"); - } - - if (keyValues.Any(k => !string.Equals(k.Label.NormalizeNull(), options.Label.NormalizeNull()))) - { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Label)}"); - } - - if (keyValues.Any(k => k.Label != null && k.Label.Contains("*"))) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); - } - - var hasKeyValueCollectionChanged = false; - var selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label, - Fields = SettingFields.ETag | SettingFields.Key + KeyFilter = keyValueSelector.KeyFilter, + LabelFilter = keyValueSelector.LabelFilter }; - // Dictionary of eTags that we write to and use for comparison - var eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - - // Fetch e-tags for prefixed key-values that can be used to detect changes - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - hasKeyValueCollectionChanged = true; - break; - } - - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - // Check for any deletions - if (eTagMap.Any()) - { - hasKeyValueCollectionChanged = true; - } + AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); - var changes = new List(); + using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - // If changes have been observed, refresh prefixed key-values - if (hasKeyValueCollectionChanged) + await foreach (Page page in pageable.AsPages(pageIterator, matchConditions).ConfigureAwait(false)) { - selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label - }; - - eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } + using Response response = page.GetRawResponse(); - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) + // Return true if the lists of etags are different + if ((!existingMatchConditionsEnumerator.MoveNext() || + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag)) && + response.Status == (int)HttpStatusCode.OK) { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); + return true; } } - return changes; + // Need to check if pages were deleted and no change was found within the new shorter list of match conditions + return existingMatchConditionsEnumerator.MoveNext(); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 8b2c488d..61572030 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -5,18 +5,11 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { - internal static class LabelFilters - { - public static readonly string Null = "\0"; - - public static readonly string Any = "*"; - } - internal static class StringExtensions { public static string NormalizeNull(this string s) { - return s == LabelFilters.Null ? null : s; + return s == LabelFilter.Null ? null : s; } public static string ToBase64String(this string s) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs index 1e8beae6..26390762 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs @@ -102,7 +102,8 @@ public FeatureFlagOptions Select(string featureFlagFilter, string labelFilter = FeatureFlagSelectors.AppendUnique(new KeyValueSelector { KeyFilter = featureFlagPrefix, - LabelFilter = labelFilter + LabelFilter = labelFilter, + IsFeatureFlagSelector = true }); return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs index f48b5220..8c696e49 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs @@ -25,7 +25,6 @@ internal class FeatureFlagTracing public bool UsesTargetingFilter { get; set; } = false; public bool UsesSeed { get; set; } = false; public bool UsesTelemetry { get; set; } = false; - public bool UsesVariantConfigurationReference { get; set; } = false; public int MaxVariants { get; set; } public bool UsesAnyFeatureFilter() @@ -35,7 +34,7 @@ public bool UsesAnyFeatureFilter() public bool UsesAnyTracingFeature() { - return UsesSeed || UsesTelemetry || UsesVariantConfigurationReference; + return UsesSeed || UsesTelemetry; } public void ResetFeatureFlagTracing() @@ -46,7 +45,6 @@ public void ResetFeatureFlagTracing() UsesTargetingFilter = false; UsesSeed = false; UsesTelemetry = false; - UsesVariantConfigurationReference = false; MaxVariants = 0; } @@ -147,16 +145,6 @@ public string CreateFeaturesString() sb.Append(RequestTracingConstants.FeatureFlagUsesSeedTag); } - if (UsesVariantConfigurationReference) - { - if (sb.Length > 0) - { - sb.Append(RequestTracingConstants.Delimiter); - } - - sb.Append(RequestTracingConstants.FeatureFlagUsesVariantConfigurationReferenceTag); - } - if (UsesTelemetry) { if (sb.Length > 0) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index aa573a1e..af4647ee 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -30,7 +30,6 @@ internal class FeatureManagementConstants public const string Parameters = "parameters"; public const string Variant = "variant"; public const string ConfigurationValue = "configuration_value"; - public const string ConfigurationReference = "configuration_reference"; public const string StatusOverride = "status_override"; public const string DefaultWhenDisabled = "default_when_disabled"; public const string DefaultWhenEnabled = "default_when_enabled"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index c29827b9..9146e175 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -188,13 +188,6 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea (string.IsNullOrEmpty(kvp.Key) ? "" : $":{kvp.Key}"), kvp.Value)); } - if (featureVariant.ConfigurationReference != null) - { - _featureFlagTracing.UsesVariantConfigurationReference = true; - - keyValues.Add(new KeyValuePair($"{variantsPath}:{FeatureManagementConstants.ConfigurationReference}", featureVariant.ConfigurationReference)); - } - if (featureVariant.StatusOverride != null) { keyValues.Add(new KeyValuePair($"{variantsPath}:{FeatureManagementConstants.StatusOverride}", featureVariant.StatusOverride)); @@ -1244,24 +1237,6 @@ private FeatureVariant ParseFeatureVariant(ref Utf8JsonReader reader, string set break; } - case FeatureManagementConstants.ConfigurationReference: - { - if (reader.Read() && reader.TokenType == JsonTokenType.String) - { - featureVariant.ConfigurationReference = reader.GetString(); - } - else if (reader.TokenType != JsonTokenType.Null) - { - throw CreateFeatureFlagFormatException( - FeatureManagementConstants.ConfigurationReference, - settingKey, - reader.TokenType.ToString(), - JsonTokenType.String.ToString()); - } - - break; - } - case FeatureManagementConstants.ConfigurationValue: { if (reader.Read()) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs index 87c5c0b1..7bfd9d6f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs @@ -11,8 +11,6 @@ internal class FeatureVariant public JsonElement ConfigurationValue { get; set; } - public string ConfigurationReference { get; set; } - public string StatusOverride { get; set; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs deleted file mode 100644 index 5cb9b83d..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class GetKeyValueChangeCollectionOptions - { - public string KeyFilter { get; set; } - public string Label { get; set; } - public bool RequestTracingEnabled { get; set; } - public RequestTracingOptions RequestTracingOptions { get; set; } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs new file mode 100644 index 00000000..08c95751 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs @@ -0,0 +1,13 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IConfigurationSettingPageIterator + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 11442dcb..4f999406 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -25,14 +25,19 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) return $"{LoggingConstants.RefreshFeatureFlagsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagReadMessage(string key, string label, string endpoint) - { - return $"{LoggingConstants.RefreshFeatureFlagRead} Key:'{key}' Label:'{label}' Endpoint:'{endpoint?.TrimEnd('/')}'"; + public static string BuildFeatureFlagsUpdatedMessage() + { + return LoggingConstants.RefreshFeatureFlagsUpdated; + } + + public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) + { + return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagUpdatedMessage(string key) + public static string BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage() { - return $"{LoggingConstants.RefreshFeatureFlagUpdated} Key:'{key}'"; + return LoggingConstants.RefreshSelectedKeyValuesAndFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index aeedd6e6..dc558788 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -19,6 +19,7 @@ + @@ -35,7 +36,7 @@ - 8.1.0-preview + 8.2.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 5491d04d..54bda1a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -24,6 +24,11 @@ public class KeyValueSelector /// public string SnapshotName { get; set; } + /// + /// A boolean that signifies whether this selector is intended to select feature flags. + /// + public bool IsFeatureFlagSelector { get; set; } + /// /// Determines whether the specified object is equal to the current object. /// diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs index 7b06535b..bd8b7582 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs @@ -63,10 +63,15 @@ internal class RequestTracingOptions /// public bool IsFailoverRequest { get; set; } = false; + /// + /// Flag to indicate whether push refresh is used. + /// + public bool IsPushRefreshUsed { get; set; } = false; + /// /// Checks whether any tracing feature is used. /// - /// True if any tracing feature is used, otherwise false. + /// true if any tracing feature is used, otherwise false. public bool UsesAnyTracingFeature() { return IsLoadBalancingEnabled || IsSignalRUsed; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs index b1b2b196..b3e12913 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/TracingUtils.cs @@ -201,6 +201,11 @@ private static string CreateCorrelationContextHeader(RequestType requestType, Re correlationContextTags.Add(RequestTracingConstants.FailoverRequestTag); } + if (requestTracingOptions.IsPushRefreshUsed) + { + correlationContextTags.Add(RequestTracingConstants.PushRefreshTag); + } + var sb = new StringBuilder(); foreach (KeyValuePair kvp in correlationContextKeyValues) diff --git a/test.ps1 b/test.ps1 index 19c80dff..9acc5054 100644 --- a/test.ps1 +++ b/test.ps1 @@ -3,7 +3,17 @@ $ErrorActionPreference = "Stop" $dotnet = & "$PSScriptRoot/build/resolve-dotnet.ps1" & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration\Tests.AzureAppConfiguration.csproj" --logger trx + +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration.AspNetCore\Tests.AzureAppConfiguration.AspNetCore.csproj" --logger trx + +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration.Functions.Worker\Tests.AzureAppConfiguration.Functions.Worker.csproj" --logger trx exit $LASTEXITCODE \ No newline at end of file diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77..c60c2a25 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -17,6 +17,11 @@ public MockResponse(int status, string reasonPhrase = null) { Status = status; ReasonPhrase = reasonPhrase; + + if (status == 200) + { + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); + } } public override int Status { get; } diff --git a/tests/Tests.AzureAppConfiguration/FailoverTests.cs b/tests/Tests.AzureAppConfiguration/FailoverTests.cs index 86ea96b9..929f9bef 100644 --- a/tests/Tests.AzureAppConfiguration/FailoverTests.cs +++ b/tests/Tests.AzureAppConfiguration/FailoverTests.cs @@ -268,10 +268,11 @@ public void FailOverTests_AutoFailover() [Fact] public void FailOverTests_ValidateEndpoints() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -285,9 +286,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager.IsValidEndpoint("azure.azconfig.bad.io")); var configClientManager2 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.appconfig.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -301,9 +301,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager2.IsValidEndpoint("azure.appconfigbad.azure.com")); var configClientManager3 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.azconfig-test.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -311,9 +310,8 @@ public void FailOverTests_ValidateEndpoints() Assert.False(configClientManager3.IsValidEndpoint("azure.azconfig.io")); var configClientManager4 = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://foobar.z1.appconfig-test.azure.com") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); @@ -325,10 +323,11 @@ public void FailOverTests_ValidateEndpoints() [Fact] public void FailOverTests_GetNoDynamicClient() { + var clientFactory = new AzureAppConfigurationClientFactory(new DefaultAzureCredential(), new ConfigurationClientOptions()); + var configClientManager = new ConfigurationClientManager( + clientFactory, new[] { new Uri("https://azure.azconfig.io") }, - new DefaultAzureCredential(), - new ConfigurationClientOptions(), true, false); diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 7e49e8ab..fb7d21ea 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -435,7 +435,6 @@ public class FeatureManagementTests }, { ""name"": ""Small"", - ""configuration_reference"": ""ShoppingCart:Small"", ""status_override"": ""Disabled"" } ], @@ -768,19 +767,24 @@ public void UsesFeatureFlags() [Fact] public async Task WatchesFeatureFlags() { + var mockResponse = new MockResponse(200); + var featureFlags = new List { _kv }; - var mockResponse = new Mock(); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -820,7 +824,7 @@ public async Task WatchesFeatureFlags() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); featureFlags.Add(_kv2); @@ -839,11 +843,12 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -852,6 +857,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -910,17 +916,20 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -977,17 +986,20 @@ public async Task SkipRefreshIfCacheNotExpired() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -1096,17 +1108,22 @@ public void QueriesFeatureFlags() } [Fact] - public async Task UsesEtagForFeatureFlagRefresh() + public async Task DoesNotUseEtagForFeatureFlagRefresh() { + var mockAsyncPageable = new MockAsyncPageable(new List { _kv }); + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(new List { _kv })); + .Callback(() => mockAsyncPageable.UpdateCollection(new List { _kv })) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1137,6 +1154,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1156,6 +1174,63 @@ public void SelectFeatureFlags() Assert.Null(config["FeatureManagement:App2_Feature2"]); } + [Fact] + public void SelectOrderDoesNotAffectLoad() + { + var mockResponse = new Mock(); + var mockClient = new Mock(MockBehavior.Strict); + + List kvCollection = new List + { + ConfigurationModelFactory.ConfigurationSetting("TestKey1", "TestValue1", "label", + eTag: new ETag("0a76e3d7-7ec1-4e37-883c-9ea6d0d89e63")), + ConfigurationModelFactory.ConfigurationSetting("TestKey2", "TestValue2", "label", + eTag: new ETag("31c38369-831f-4bf1-b9ad-79db56c8b989")) + }; + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + List settingCollection; + + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + settingCollection = _featureFlagCollection; + } + else + { + settingCollection = kvCollection; + } + + var copy = new List(); + var newSetting = settingCollection.FirstOrDefault(s => (s.Key == selector.KeyFilter && s.Label == selector.LabelFilter)); + if (newSetting != null) + copy.Add(TestHelpers.CloneSetting(newSetting)); + return new MockAsyncPageable(copy); + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.UseFeatureFlags(ff => + { + ff.Select("App1_Feature1", "App1_Label"); + ff.Select("App2_Feature1", "App2_Label"); + }); + options.Select("TestKey1", "label"); + options.Select("TestKey2", "label"); + }) + .Build(); + + Assert.Equal("True", config["FeatureManagement:App1_Feature1"]); + Assert.Equal("False", config["FeatureManagement:App2_Feature1"]); + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue2", config["TestKey2"]); + } + [Fact] public void TestNullAndMissingValuesForConditions() { @@ -1486,18 +1561,19 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || - (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList()); - }); + (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1656,18 +1732,18 @@ public async Task SelectAndRefreshSingleFeatureFlag() var label1 = "App1_Label"; IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => - s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList()); - }); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => + s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1720,8 +1796,17 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetIfChanged); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetTestKey); string informationalInvocation = ""; string verboseInvocation = ""; @@ -1745,6 +1830,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1753,10 +1839,10 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "myFeature1", + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature2", value: @" { - ""id"": ""MyFeature"", + ""id"": ""MyFeature2"", ""description"": ""The new beta version of our web site."", ""display_name"": ""Beta Feature"", ""enabled"": true, @@ -1771,21 +1857,19 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); Assert.Null(config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); } [Fact] @@ -1796,8 +1880,11 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1821,6 +1908,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { @@ -1832,7 +1920,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .Build(); Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); - FirstKeyValue.Value = "newValue1"; + FirstKeyValue = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -1870,9 +1958,11 @@ public async Task MapTransformFeatureFlagWithRefresh() IConfigurationRefresher refresher = null; var featureFlags = new List { _kv }; var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1925,7 +2015,7 @@ public async Task MapTransformFeatureFlagWithRefresh() Assert.Equal("TestValue1", config["TestKey1"]); Assert.Equal("NoUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - FirstKeyValue.Value = "newValue1"; + FirstKeyValue = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", value: @" @@ -1945,7 +2035,7 @@ public async Task MapTransformFeatureFlagWithRefresh() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -1976,7 +2066,6 @@ public void WithVariants() Assert.Equal("Big", config["feature_management:feature_flags:0:variants:0:name"]); Assert.Equal("600px", config["feature_management:feature_flags:0:variants:0:configuration_value"]); Assert.Equal("Small", config["feature_management:feature_flags:0:variants:1:name"]); - Assert.Equal("ShoppingCart:Small", config["feature_management:feature_flags:0:variants:1:configuration_reference"]); Assert.Equal("Disabled", config["feature_management:feature_flags:0:variants:1:status_override"]); Assert.Equal("Small", config["feature_management:feature_flags:0:allocation:default_when_disabled"]); Assert.Equal("Small", config["feature_management:feature_flags:0:allocation:default_when_enabled"]); @@ -2190,7 +2279,7 @@ public void ThrowsOnIncorrectJsonTypes() var settings = new List() { CreateFeatureFlag("Feature1", variantsJsonString: @"[{""name"": 1}]"), - CreateFeatureFlag("Feature2", variantsJsonString: @"[{""configuration_reference"": true}]"), + CreateFeatureFlag("Feature2", requirementType: "2"), CreateFeatureFlag("Feature3", variantsJsonString: @"[{""status_override"": []}]"), CreateFeatureFlag("Feature4", seed: "{}"), CreateFeatureFlag("Feature5", defaultWhenDisabled: "5"), @@ -2205,8 +2294,7 @@ public void ThrowsOnIncorrectJsonTypes() CreateFeatureFlag("Feature14", telemetryEnabled: "14"), CreateFeatureFlag("Feature15", telemetryMetadataJsonString: @"{""key"": 15}"), CreateFeatureFlag("Feature16", clientFiltersJsonString: @"[{""name"": 16}]"), - CreateFeatureFlag("Feature17", clientFiltersJsonString: @"{""key"": [{""name"": ""name"", ""parameters"": 17}]}"), - CreateFeatureFlag("Feature18", requirementType: "18") + CreateFeatureFlag("Feature17", clientFiltersJsonString: @"{""key"": [{""name"": ""name"", ""parameters"": 17}]}") }; var mockResponse = new Mock(); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index 06c88040..274bfe7f 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -505,9 +505,9 @@ public void DoesNotThrowKeyVaultExceptionWhenProviderIsOptional() .Returns(new MockAsyncPageable(new List { _kv })); var mockKeyValueAdapter = new Mock(MockBehavior.Strict); - mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(_kv)) + mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(It.IsAny())) .Returns(true); - mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(_kv, It.IsAny(), It.IsAny(), It.IsAny())) + mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Throws(new KeyVaultReferenceException("Key vault error", null)); mockKeyValueAdapter.Setup(adapter => adapter.OnChangeDetected(null)); mockKeyValueAdapter.Setup(adapter => adapter.OnConfigUpdated()); @@ -743,7 +743,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal(_secretValue, config[_kv.Key]); // Update sentinel key-value - sentinelKv.Value = "Value2"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "Value2"); Thread.Sleep(refreshInterval); await refresher.RefreshAsync(); @@ -815,7 +815,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal(_secretValue, config[_kv.Key]); // Update sentinel key-value to trigger refresh operation - sentinelKv.Value = "Value2"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "Value2"); Thread.Sleep(refreshInterval); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/LoggingTests.cs b/tests/Tests.AzureAppConfiguration/LoggingTests.cs index 547c65bd..6fcec29f 100644 --- a/tests/Tests.AzureAppConfiguration/LoggingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoggingTests.cs @@ -239,7 +239,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("SentinelValue", config["SentinelKey"]); // Update sentinel key-value to trigger refreshAll operation - sentinelKv.Value = "UpdatedSentinelValue"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "UpdatedSentinelValue"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -377,7 +377,7 @@ public async Task ValidateFailoverToDifferentEndpointMessageLoggedAfterFailover( .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -389,7 +389,7 @@ public async Task ValidateFailoverToDifferentEndpointMessageLoggedAfterFailover( .Throws(new RequestFailedException(HttpStatusCodes.TooManyRequests, "Too many requests")); mockClient2.Setup(c => c.ToString()).Returns("client"); - FirstKeyValue.Value = "TestValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "TestValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -431,7 +431,7 @@ public async Task ValidateConfigurationUpdatedSuccessLoggedDuringRefresh() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -480,7 +480,7 @@ public async Task ValidateCorrectEndpointLoggedOnConfigurationUpdate() }) .Build(); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -528,7 +528,7 @@ public async Task ValidateCorrectKeyValueLoggedDuringRefresh() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/MapTests.cs b/tests/Tests.AzureAppConfiguration/MapTests.cs index 623ae477..cdf15f4e 100644 --- a/tests/Tests.AzureAppConfiguration/MapTests.cs +++ b/tests/Tests.AzureAppConfiguration/MapTests.cs @@ -181,7 +181,7 @@ public async Task MapTransformWithRefresh() Assert.Equal("TestValue1 mapped first", config["TestKey1"]); Assert.Equal("TestValue2 second", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -232,8 +232,8 @@ public async Task MapTransformSettingKeyWithRefresh() Assert.Null(config["TestKey1"]); Assert.Equal("TestValue2", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -282,8 +282,8 @@ public async Task MapTransformSettingLabelWithRefresh() Assert.Equal("TestValue1 changed", config["TestKey1"]); Assert.Equal("TestValue2 changed", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -523,8 +523,8 @@ public async Task MapTransformSettingKeyWithLogAndRefresh() Assert.Null(config["TestKey1"]); Assert.Equal("TestValue2", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs b/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs index c4c7c38c..41e265d2 100644 --- a/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs @@ -347,7 +347,7 @@ public async Task RefreshAsyncUpdatesConfig() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); refresher.ProcessPushNotification(_pushNotificationList.First(), TimeSpan.FromSeconds(0)); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 6edc1a9a..e64b5184 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -7,6 +7,7 @@ using Azure.Identity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Logging.Abstractions; using Moq; using System; @@ -208,10 +209,10 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -221,7 +222,6 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() [Fact] public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() { - var keyValueCollection = new List(_kvCollection); IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -244,10 +244,10 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -259,7 +259,6 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() [Fact] public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() { - var keyValueCollection = new List(_kvCollection); IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -282,10 +281,10 @@ public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -353,11 +352,11 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.First().Value = "newValue"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue"); keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -426,12 +425,12 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ElementAt(0).Value = "newValue1"; - keyValueCollection.ElementAt(1).Value = "newValue2"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue1"); + keyValueCollection[1] = TestHelpers.ChangeValue(keyValueCollection[1], "newValue2"); keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -444,32 +443,33 @@ Response GetIfChanged(ConfigurationSetting setting, bool o } [Fact] - public async void RefreshTests_SingleServerCallOnSimultaneousMultipleRefresh() + public async Task RefreshTests_SingleServerCallOnSimultaneousMultipleRefresh() { var keyValueCollection = new List(_kvCollection); var requestCount = 0; var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + // Define delay for async operations + var operationDelay = TimeSpan.FromSeconds(6); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) .Returns(() => { requestCount++; - Thread.Sleep(6000); - var copy = new List(); foreach (var setting in keyValueCollection) { copy.Add(TestHelpers.CloneSetting(setting)); }; - return new MockAsyncPageable(copy); + return new MockAsyncPageable(copy, operationDelay); }); - Response GetIfChanged(ConfigurationSetting setting, bool onlyIfChanged, CancellationToken cancellationToken) + async Task> GetIfChanged(ConfigurationSetting setting, bool onlyIfChanged, CancellationToken cancellationToken) { requestCount++; - Thread.Sleep(6000); + await Task.Delay(operationDelay, cancellationToken); var newSetting = keyValueCollection.FirstOrDefault(s => s.Key == setting.Key && s.Label == setting.Label); var unchanged = (newSetting.Key == setting.Key && newSetting.Label == setting.Label && newSetting.Value == setting.Value); @@ -478,7 +478,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o } mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((Func>)GetIfChanged); + .Returns((Func>>)GetIfChanged); IConfigurationRefresher refresher = null; @@ -499,7 +499,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue1", config["TestKey1"]); Assert.Equal(1, requestCount); - keyValueCollection.First().Value = "newValue"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue"); // Simulate simultaneous refresh calls with expired cache from multiple threads var task1 = Task.Run(() => WaitAndRefresh(refresher, 1500)); @@ -513,7 +513,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o } [Fact] - public void RefreshTests_RefreshAsyncThrowsOnRequestFailedException() + public async Task RefreshTests_RefreshAsyncThrowsOnRequestFailedException() { IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -540,7 +540,7 @@ public void RefreshTests_RefreshAsyncThrowsOnRequestFailedException() .Throws(new RequestFailedException("Request failed.")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); Action action = () => refresher.RefreshAsync().Wait(); Assert.Throws(action); @@ -576,7 +576,7 @@ public async Task RefreshTests_TryRefreshAsyncReturnsFalseOnRequestFailedExcepti .Throws(new RequestFailedException("Request failed.")); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool result = await refresher.TryRefreshAsync(); Assert.False(result); @@ -606,10 +606,10 @@ public async Task RefreshTests_TryRefreshAsyncUpdatesConfigurationAndReturnsTrue .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool result = await refresher.TryRefreshAsync(); Assert.True(result); @@ -652,13 +652,13 @@ public async Task RefreshTests_TryRefreshAsyncReturnsFalseForAuthenticationFaile FirstKeyValue.Value = "newValue"; // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); // First call to GetConfigurationSettingAsync does not throw Assert.True(await refresher.TryRefreshAsync()); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); // Second call to GetConfigurationSettingAsync throws KeyVaultReferenceException Assert.False(await refresher.TryRefreshAsync()); @@ -702,10 +702,10 @@ Response GetIfChanged(ConfigurationSetting setting, bool o .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await Assert.ThrowsAsync(async () => await refresher.RefreshAsync() @@ -749,7 +749,7 @@ public async Task RefreshTests_UpdatesAllSettingsIfInitialLoadFails() Assert.Null(configuration["TestKey3"]); // Make sure MinBackoffDuration has ended - Thread.Sleep(100); + await Task.Delay(100); // Act await Assert.ThrowsAsync(async () => @@ -764,7 +764,7 @@ await Assert.ThrowsAsync(async () => Assert.Null(configuration["TestKey3"]); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -823,10 +823,10 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + keyValueCollection = keyValueCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool firstRefreshResult = await refresher.TryRefreshAsync(); Assert.False(firstRefreshResult); @@ -836,7 +836,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue3", config["TestKey3"]); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); bool secondRefreshResult = await refresher.TryRefreshAsync(); Assert.True(secondRefreshResult); @@ -874,10 +874,10 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel2", config["TestKeyWithMultipleLabels"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -890,8 +890,7 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire [Fact] public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfig() { - var keyValueCollection = new List(_kvCollection); - ConfigurationSetting refreshRegisteredSetting = keyValueCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); + ConfigurationSetting refreshRegisteredSetting = _kvCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); var mockClient = GetMockConfigurationClient(); IConfigurationRefresher refresher = null; @@ -916,10 +915,10 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel2", config["TestKeyWithMultipleLabels"]); - refreshRegisteredSetting.Value = "UpdatedValueForLabel1"; + _kvCollection[_kvCollection.IndexOf(refreshRegisteredSetting)] = TestHelpers.ChangeValue(refreshRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -933,8 +932,7 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi [Fact] public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() { - var keyValueCollection = new List(_kvCollection); - ConfigurationSetting refreshAllRegisteredSetting = keyValueCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); + ConfigurationSetting refreshAllRegisteredSetting = _kvCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); var mockClient = GetMockConfigurationClient(); IConfigurationRefresher refresher = null; @@ -959,10 +957,10 @@ public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel1", config["TestKeyWithMultipleLabels"]); - refreshAllRegisteredSetting.Value = "UpdatedValueForLabel1"; + _kvCollection[_kvCollection.IndexOf(refreshAllRegisteredSetting)] = TestHelpers.ChangeValue(refreshAllRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); await refresher.RefreshAsync(); @@ -1023,7 +1021,7 @@ public void RefreshTests_ConfigureRefreshThrowsOnNoRegistration() } [Fact] - public void RefreshTests_RefreshIsCancelled() + public async Task RefreshTests_RefreshIsCancelled() { IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -1046,7 +1044,7 @@ public void RefreshTests_RefreshIsCancelled() FirstKeyValue.Value = "newValue1"; // Wait for the cache to expire - Thread.Sleep(1500); + await Task.Delay(1500); using var cancellationSource = new CancellationTokenSource(); cancellationSource.Cancel(); @@ -1056,6 +1054,170 @@ public void RefreshTests_RefreshIsCancelled() Assert.Equal("TestValue1", config["TestKey1"]); } + [Fact] + public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var mockAsyncPageable = new MockAsyncPageable(_kvCollection); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Callback(() => mockAsyncPageable.UpdateCollection(_kvCollection)) + .Returns(mockAsyncPageable); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*", "label"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue3", config["TestKey3"]); + FirstKeyValue.Value = "newValue1"; + _kvCollection[2].Value = "newValue3"; + + // Wait for the cache to expire + await Task.Delay(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("newValue3", config["TestKey3"]); + + _kvCollection.RemoveAt(2); + + // Wait for the cache to expire + await Task.Delay(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Null(config["TestKey3"]); + } + + [Fact] + public async Task RefreshTests_RegisterAllRefreshesFeatureFlags() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var featureFlags = new List { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""SuperUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) + }; + + var mockAsyncPageableKv = new MockAsyncPageable(_kvCollection); + + var mockAsyncPageableFf = new MockAsyncPageable(featureFlags); + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + mockAsyncPageableFf.UpdateCollection(featureFlags); + + return mockAsyncPageableFf; + } + + mockAsyncPageableKv.UpdateCollection(_kvCollection); + + return mockAsyncPageableKv; + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*", "label"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + options.UseFeatureFlags(); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newValue1"; + featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""AllUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + + // Wait for the cache to expire + await Task.Delay(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newerValue1"; + featureFlags.RemoveAt(0); + + // Wait for the cache to expire + await Task.Delay(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newerValue1", config["TestKey1"]); + Assert.Null(config["FeatureManagement:MyFeature"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index bc7989b2..9fd3f388 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Core; +using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; using Microsoft.Extensions.Logging; @@ -10,6 +11,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -115,6 +117,11 @@ public static ConfigurationSetting CloneSetting(ConfigurationSetting setting) return ConfigurationModelFactory.ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ContentType, setting.ETag, setting.LastModified); } + public static ConfigurationSetting ChangeValue(ConfigurationSetting setting, string value) + { + return ConfigurationModelFactory.ConfigurationSetting(setting.Key, value, setting.Label, setting.ContentType, new ETag(Guid.NewGuid().ToString()), setting.LastModified); + } + public static List LoadJsonSettingsFromFile(string path) { List _kvCollection = new List(); @@ -155,19 +162,74 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection; + private readonly List _collection = new List(); + private int _status; + private readonly TimeSpan? _delay; - public MockAsyncPageable(List collection) + public MockAsyncPageable(List collection, TimeSpan? delay = null) { - _collection = collection; + foreach (ConfigurationSetting setting in collection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + + _status = 200; + _delay = delay; } -#pragma warning disable 1998 - public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) -#pragma warning restore 1998 + public void UpdateCollection(List newCollection) { - yield return Page.FromValues(_collection, null, new Mock().Object); + if (_collection.Count() == newCollection.Count() && + _collection.All(setting => newCollection.Any(newSetting => + setting.Key == newSetting.Key && + setting.Value == newSetting.Value && + setting.Label == newSetting.Label && + setting.ETag == newSetting.ETag))) + { + _status = 304; + } + else + { + _status = 200; + + _collection.Clear(); + + foreach (ConfigurationSetting setting in newCollection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + _collection.Add(newSetting); + } + } + } + + public override async IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) + { + if (_delay.HasValue) + { + await Task.Delay(_delay.Value); + } + + yield return Page.FromValues(_collection, null, new MockResponse(_status)); + } + } + + internal class MockConfigurationSettingPageIterator : IConfigurationSettingPageIterator + { + public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(); + } + + public IAsyncEnumerable> IteratePages(AsyncPageable pageable) + { + return pageable.AsPages(); } } @@ -182,7 +244,7 @@ public MockPageable(List collection) public override IEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } }