[Package Renames] Add feature flag to disable popularity transfers (#773

)

Adds the feature flag `Search.TransferPopularity` that, if disabled, removes all popularity transfers on the search index. Note that download overrides are still applied even if the popularity transfer feature flag is disabled.

Addresses NuGet/NuGetGallery#7944
The feature flag staleness monitoring is tracked by NuGet/NuGetGallery#7966

src/NuGet.Jobs.Db2AzureSearch/Job.cs

@@ -3,7 +3,6 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Options;
 using NuGet.Services.AzureSearch;
 using NuGet.Services.AzureSearch.AuxiliaryFiles;
 using NuGet.Services.AzureSearch.Db2AzureSearch;

src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateDownloadsCommand.cs

@@ -37,6 +37,7 @@ public class UpdateDownloadsCommand : IAzureSearchCommand
         private readonly ISearchIndexActionBuilder _indexActionBuilder;
         private readonly Func<IBatchPusher> _batchPusherFactory;
         private readonly ISystemTime _systemTime;
+        private readonly IFeatureFlagService _featureFlags;
         private readonly IOptionsSnapshot<Auxiliary2AzureSearchConfiguration> _options;
         private readonly IAzureSearchTelemetryService _telemetryService;
         private readonly ILogger<Auxiliary2AzureSearchCommand> _logger;
@@ -53,6 +54,7 @@ public UpdateDownloadsCommand(
             ISearchIndexActionBuilder indexActionBuilder,
             Func<IBatchPusher> batchPusherFactory,
             ISystemTime systemTime,
+            IFeatureFlagService featureFlags,
             IOptionsSnapshot<Auxiliary2AzureSearchConfiguration> options,
             IAzureSearchTelemetryService telemetryService,
             ILogger<Auxiliary2AzureSearchCommand> logger)
@@ -67,6 +69,7 @@ public UpdateDownloadsCommand(
             _indexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder));
             _batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory));
             _systemTime = systemTime ?? throw new ArgumentNullException(nameof(systemTime));
+            _featureFlags = featureFlags ?? throw new ArgumentNullException(nameof(featureFlags));
             _options = options ?? throw new ArgumentNullException(nameof(options));
             _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -132,7 +135,7 @@ private async Task<bool> PushIndexChangesAsync()
 
             // The "new" data is the latest popularity transfers data from the database.
             _logger.LogInformation("Fetching new popularity transfer data from database.");
-            var newTransfers = await _databaseFetcher.GetPackageIdToPopularityTransfersAsync();
+            var newTransfers = await GetPopularityTransfersAsync();
 
             _logger.LogInformation("Fetching new download overrides from blob storage.");
             var downloadOverrides = await _auxiliaryFileClient.LoadDownloadOverridesAsync();
@@ -171,6 +174,19 @@ await _popularityTransferDataClient.ReplaceLatestIndexedAsync(
             return true;
         }
 
+        private async Task<SortedDictionary<string, SortedSet<string>>> GetPopularityTransfersAsync()
+        {
+            if (!_featureFlags.IsPopularityTransferEnabled())
+            {
+                _logger.LogWarning(
+                    "Popularity transfers feature flag is disabled. " +
+                    "All popularity transfers will be removed.");
+                return new SortedDictionary<string, SortedSet<string>>(StringComparer.OrdinalIgnoreCase);
+            }
+
+            return await _databaseFetcher.GetPackageIdToPopularityTransfersAsync();
+        }
+
         private void ApplyDownloadTransfers(
             DownloadData newData,
             SortedDictionary<string, SortedSet<string>> oldTransfers,

src/NuGet.Services.AzureSearch/AzureSearchJob.cs

@@ -8,16 +8,22 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Rest;
 using NuGet.Jobs;
+using NuGet.Jobs.Validation;
 
 namespace NuGet.Services.AzureSearch
 {
     public abstract class AzureSearchJob<T> : JsonConfigurationJob where T : IAzureSearchCommand
     {
+        private const string FeatureFlagConfigurationSectionName = "FeatureFlags";
+
         public override async Task Run()
         {
             ServicePointManager.DefaultConnectionLimit = 64;
             ServicePointManager.MaxServicePointIdleTime = 10000;
 
+            var featureFlagRefresher = _serviceProvider.GetRequiredService<IFeatureFlagRefresher>();
+            await featureFlagRefresher.StartIfConfiguredAsync();
+
             var tracingInterceptor = _serviceProvider.GetRequiredService<IServiceClientTracingInterceptor>();
             try
             {
@@ -30,6 +36,8 @@ public override async Task Run()
             {
                 ServiceClientTracing.RemoveTracingInterceptor(tracingInterceptor);
             }
+
+            await featureFlagRefresher.StopAndWaitAsync();
         }
 
         protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
@@ -40,6 +48,8 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild
         protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
             services.AddAzureSearch(GlobalTelemetryDimensions);
+
+            services.Configure<FeatureFlagConfiguration>(configurationRoot.GetSection(FeatureFlagConfigurationSectionName));
         }
     }
 }

src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistrationProducer.cs

@@ -23,6 +23,7 @@ public class NewPackageRegistrationProducer : INewPackageRegistrationProducer
         private readonly IAuxiliaryFileClient _auxiliaryFileClient;
         private readonly IDatabaseAuxiliaryDataFetcher _databaseFetcher;
         private readonly IDownloadTransferrer _downloadTransferrer;
+        private readonly IFeatureFlagService _featureFlags;
         private readonly IOptionsSnapshot<Db2AzureSearchConfiguration> _options;
         private readonly IOptionsSnapshot<Db2AzureSearchDevelopmentConfiguration> _developmentOptions;
         private readonly ILogger<NewPackageRegistrationProducer> _logger;
@@ -32,6 +33,7 @@ public NewPackageRegistrationProducer(
             IAuxiliaryFileClient auxiliaryFileClient,
             IDatabaseAuxiliaryDataFetcher databaseFetcher,
             IDownloadTransferrer downloadTransferrer,
+            IFeatureFlagService featureFlags,
             IOptionsSnapshot<Db2AzureSearchConfiguration> options,
             IOptionsSnapshot<Db2AzureSearchDevelopmentConfiguration> developmentOptions,
             ILogger<NewPackageRegistrationProducer> logger)
@@ -40,6 +42,7 @@ public NewPackageRegistrationProducer(
             _auxiliaryFileClient = auxiliaryFileClient ?? throw new ArgumentNullException(nameof(auxiliaryFileClient));
             _databaseFetcher = databaseFetcher ?? throw new ArgumentNullException(nameof(databaseFetcher));
             _downloadTransferrer = downloadTransferrer ?? throw new ArgumentNullException(nameof(downloadTransferrer));
+            _featureFlags = featureFlags ?? throw new ArgumentNullException(nameof(featureFlags));
             _options = options ?? throw new ArgumentNullException(nameof(options));
             _developmentOptions = developmentOptions ?? throw new ArgumentNullException(nameof(developmentOptions));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -64,7 +67,7 @@ public async Task<InitialAuxiliaryData> ProduceWorkAsync(
             // auxiliary file.
             var downloads = await _auxiliaryFileClient.LoadDownloadDataAsync();
 
-            var popularityTransfers = await _databaseFetcher.GetPackageIdToPopularityTransfersAsync();
+            var popularityTransfers = await GetPopularityTransfersAsync();
             var downloadOverrides = await _auxiliaryFileClient.LoadDownloadOverridesAsync();
 
             // Apply changes from popularity transfers and download overrides.
@@ -160,6 +163,19 @@ private bool ShouldWait(ConcurrentBag<NewPackageRegistration> allWork, bool log)
             return false;
         }
 
+        private async Task<SortedDictionary<string, SortedSet<string>>> GetPopularityTransfersAsync()
+        {
+            if (!_featureFlags.IsPopularityTransferEnabled())
+            {
+                _logger.LogWarning(
+                    "Popularity transfers feature flag is disabled. " +
+                    "Popularity transfers will be ignored.");
+                return new SortedDictionary<string, SortedSet<string>>(StringComparer.OrdinalIgnoreCase);
+            }
+
+            return await _databaseFetcher.GetPackageIdToPopularityTransfersAsync();
+        }
+
         private Dictionary<string, long> GetTransferredDownloads(
             DownloadData downloads,
             SortedDictionary<string, SortedSet<string>> popularityTransfers,

src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs

@@ -29,6 +29,8 @@ public static class DependencyInjectionExtensions
     {
         public static ContainerBuilder AddAzureSearch(this ContainerBuilder containerBuilder)
         {
+            containerBuilder.AddFeatureFlags();
+
             /// Here, we register services that depend on an interface that there are multiple implementations.
 
             /// There are multiple implementations of <see cref="ISearchServiceClientWrapper"/>.
@@ -226,6 +228,9 @@ public static IServiceCollection AddAzureSearch(
         {
             services.AddV3(telemetryGlobalDimensions);
 
+            services.AddFeatureFlags();
+            services.AddTransient<IFeatureFlagService, FeatureFlagService>();
+
             services
                 .AddTransient<ISearchServiceClient>(p =>
                 {

src/NuGet.Services.AzureSearch/FeatureFlagService.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using NuGet.Services.FeatureFlags;
+
+namespace NuGet.Services.AzureSearch
+{
+    public class FeatureFlagService : IFeatureFlagService
+    {
+        private const string SearchPrefix = "Search.";
+
+        private readonly IFeatureFlagClient _featureFlagClient;
+
+        public FeatureFlagService(IFeatureFlagClient featureFlagClient)
+        {
+            _featureFlagClient = featureFlagClient ?? throw new ArgumentNullException(nameof(featureFlagClient));
+        }
+
+        public bool IsPopularityTransferEnabled()
+        {
+            return _featureFlagClient.IsEnabled(SearchPrefix + "TransferPopularity", defaultValue: true);
+        }
+    }
+}

src/NuGet.Services.AzureSearch/IFeatureFlagService.cs

@@ -0,0 +1,10 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.AzureSearch
+{
+    public interface IFeatureFlagService
+    {
+        bool IsPopularityTransferEnabled();
+    }
+}

src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj

@@ -81,11 +81,13 @@
     <Compile Include="Db2AzureSearch\Db2AzureSearchDevelopmentConfiguration.cs" />
     <Compile Include="Db2AzureSearch\InitialAuxiliaryData.cs" />
     <Compile Include="DownloadTransferrer.cs" />
+    <Compile Include="FeatureFlagService.cs" />
     <Compile Include="IAzureSearchTelemetryService.cs" />
     <Compile Include="IBaseDocumentBuilder.cs" />
     <Compile Include="IAzureSearchCommand.cs" />
     <Compile Include="IDatabaseAuxiliaryDataFetcher.cs" />
     <Compile Include="IDownloadTransferrer.cs" />
+    <Compile Include="IFeatureFlagService.cs" />
     <Compile Include="ISearchIndexActionBuilder.cs" />
     <Compile Include="JobOutcome.cs" />
     <Compile Include="Models\IUpdatedDocument.cs" />

src/NuGet.Services.V3/DependencyInjectionExtensions.cs

@@ -1,22 +1,35 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Collections.Generic;
 using System.Net;
 using System.Net.Http;
+using Autofac;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Microsoft.WindowsAzure.Storage.RetryPolicies;
 using NuGet.Jobs.Validation;
 using NuGet.Protocol.Catalog;
 using NuGet.Protocol.Registration;
+using NuGet.Services.FeatureFlags;
 using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
+using NuGetGallery;
 using NuGetGallery.Diagnostics;
+using NuGetGallery.Features;
 
 namespace NuGet.Services.V3
 {
     public static class DependencyInjectionExtensions
     {
+        private const string FeatureFlagBindingKey = nameof(FeatureFlagBindingKey);
+
+        private static readonly TimeSpan FeatureFlagServerTimeout = TimeSpan.FromSeconds(30);
+        private static readonly TimeSpan FeatureFlagMaxExecutionTime = TimeSpan.FromMinutes(10);
+
         public static IServiceCollection AddV3(this IServiceCollection services, IDictionary<string, string> telemetryGlobalDimensions)
         {
             services
@@ -50,5 +63,60 @@ public static IServiceCollection AddV3(this IServiceCollection services, IDictio
 
             return services;
         }
+
+        public static void AddFeatureFlags(this ContainerBuilder containerBuilder)
+        {
+            containerBuilder
+                .Register(c =>
+                {
+                    var options = c.Resolve<IOptionsSnapshot<FeatureFlagConfiguration>>();
+                    var requestOptions = new BlobRequestOptions
+                    {
+                        ServerTimeout = FeatureFlagServerTimeout,
+                        MaximumExecutionTime = FeatureFlagMaxExecutionTime,
+                        LocationMode = LocationMode.PrimaryThenSecondary,
+                        RetryPolicy = new ExponentialRetry(),
+                    };
+
+                    return new CloudBlobClientWrapper(
+                        options.Value.ConnectionString,
+                        requestOptions);
+                })
+                .Keyed<ICloudBlobClient>(FeatureFlagBindingKey);
+
+            containerBuilder
+                .Register(c => new CloudBlobCoreFileStorageService(
+                    c.ResolveKeyed<ICloudBlobClient>(FeatureFlagBindingKey),
+                    c.Resolve<IDiagnosticsService>(),
+                    c.Resolve<ICloudBlobContainerInformationProvider>()))
+                .Keyed<ICoreFileStorageService>(FeatureFlagBindingKey);
+
+            containerBuilder
+                .Register(c => new FeatureFlagFileStorageService(
+                    c.ResolveKeyed<ICoreFileStorageService>(FeatureFlagBindingKey)))
+                .As<IFeatureFlagStorageService>();
+        }
+
+        public static IServiceCollection AddFeatureFlags(this IServiceCollection services)
+        {
+	        services
+                .AddTransient(p =>
+                {
+                    var options = p.GetRequiredService<IOptionsSnapshot<FeatureFlagConfiguration>>();
+                    return new FeatureFlagOptions
+                    {
+                        RefreshInterval = options.Value.RefreshInternal,
+                    };
+                });
+
+            services.AddTransient<IFeatureFlagClient, FeatureFlagClient>();
+            services.AddTransient<IFeatureFlagTelemetryService, V3TelemetryService>();
+            services.AddTransient<ICloudBlobContainerInformationProvider, GalleryCloudBlobContainerInformationProvider>();
+
+            services.AddSingleton<IFeatureFlagCacheService, FeatureFlagCacheService>();
+            services.AddSingleton<IFeatureFlagRefresher, FeatureFlagRefresher>();
+
+            return services;
+        }
     }
 }

src/NuGet.Services.V3/V3TelemetryService.cs

@@ -3,11 +3,12 @@
 
 using System;
 using System.Collections.Generic;
+using NuGet.Services.FeatureFlags;
 using NuGet.Services.Logging;
 
 namespace NuGet.Services.V3
 {
-    public class V3TelemetryService : IV3TelemetryService
+    public class V3TelemetryService : IV3TelemetryService, IFeatureFlagTelemetryService
     {
         private const string Prefix = "V3.";
 
@@ -27,5 +28,12 @@ public IDisposable TrackCatalogLeafDownloadBatch(int count)
                     { "Count", count.ToString() },
                 });
         }
+
+        public void TrackFeatureFlagStaleness(TimeSpan staleness)
+        {
+            _telemetryClient.TrackMetric(
+                Prefix + "FeatureFlagStalenessSeconds",
+                staleness.TotalSeconds);
+        }
     }
 }