[ReleasePrep][2024.08.20] RI dev to main

build/common.ps1

@@ -161,7 +161,6 @@ Function Invoke-BuildStep {
         if ($env:TF_BUILD) {
             Write-Output "##[group]$BuildStep"
         }
-
         Trace-Log "[BEGIN] $BuildStep"
         $sw = [Diagnostics.Stopwatch]::StartNew()
         $completed = $false
@@ -173,6 +172,9 @@ Function Invoke-BuildStep {
         finally {
             $sw.Stop()
             Reset-Colors
+            if ($env:TF_BUILD) {
+                Write-Output "##[endgroup]"
+            }
             if ($completed) {
                 Trace-Log "[DONE +$(Format-ElapsedTime $sw.Elapsed)] $BuildStep"
             }
@@ -184,14 +186,10 @@ Function Invoke-BuildStep {
                     Error-Log "[FAILED +$(Format-ElapsedTime $sw.Elapsed)] $BuildStep"
                 }
             }
-
-            if ($env:TF_BUILD) {
-                Write-Output "##[endgroup]"
-            }
         }
     }
     else {
-        Warning-Log "[SKIP] $BuildStep"
+        Trace-Log "[SKIP] $BuildStep"
     }
 }
 

src/Catalog/Downloads/DownloadsV1JsonClient.cs

@@ -1,11 +1,13 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
 using System.Diagnostics;
 using System.IO;
-using System.Net.Http;
 using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 using NuGet.Services.Metadata.Catalog.Helpers;
@@ -14,46 +16,43 @@ namespace NuGet.Services.Metadata.Catalog
 {
     public class DownloadsV1JsonClient : IDownloadsV1JsonClient
     {
-        private readonly HttpClient _httpClient;
+        private readonly BlobClient _blobClient;
         private readonly ILogger<DownloadsV1JsonClient> _logger;
 
-        public DownloadsV1JsonClient(HttpClient httpClient, ILogger<DownloadsV1JsonClient> logger)
+        public DownloadsV1JsonClient(BlobClient blobClient, ILogger<DownloadsV1JsonClient> logger)
         {
-            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+            _blobClient = blobClient ?? throw new ArgumentNullException(nameof(blobClient));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task<DownloadData> ReadAsync(string url)
+        public async Task<DownloadData> ReadAsync()
         {
             var downloadData = new DownloadData();
-            await ReadAsync(url, downloadData.SetDownloadCount);
+            await ReadAsync(downloadData.SetDownloadCount);
             return downloadData;
         }
 
-        public async Task ReadAsync(string url, AddDownloadCount addCount)
+        public async Task ReadAsync(AddDownloadCount addCount)
         {
             var stopwatch = Stopwatch.StartNew();
             var packageCount = 0;
-            
+
             await Retry.IncrementalAsync(
                 async () =>
                 {
-                    _logger.LogInformation("Attempting to download {Url}", url);
-                    using (var response = await _httpClient.GetAsync(url))
+                    _logger.LogInformation("Attempting to download {Url}", _blobClient.Uri.GetLeftPart(UriPartial.Path));
+                    using (BlobDownloadStreamingResult result = await _blobClient.DownloadStreamingAsync())
+                    using (var textReader = new StreamReader(result.Content))
+                    using (var jsonReader = new JsonTextReader(textReader))
                     {
-                        response.EnsureSuccessStatusCode();
-                        using (var textReader = new StreamReader(await response.Content.ReadAsStreamAsync()))
-                        using (var jsonReader = new JsonTextReader(textReader))
+                        DownloadsV1Reader.Load(jsonReader, (id, version, count) =>
                         {
-                            DownloadsV1Reader.Load(jsonReader, (id, version, count) =>
-                            {
-                                packageCount++;
-                                addCount(id, version, count);
-                            });
-                        }
+                            packageCount++;
+                            addCount(id, version, count);
+                        });
                     }
                 },
-                ex => ex is HttpRequestException,
+                ex => ex is RequestFailedException,
                 maxRetries: 5,
                 initialWaitInterval: TimeSpan.Zero,
                 waitIncrement: TimeSpan.FromSeconds(20));

src/Catalog/Downloads/IDownloadsV1JsonClient.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System.Threading.Tasks;
@@ -9,7 +9,7 @@ namespace NuGet.Services.Metadata.Catalog
 
     public interface IDownloadsV1JsonClient
     {
-        Task<DownloadData> ReadAsync(string url);
-        Task ReadAsync(string url, AddDownloadCount addCount);
+        Task<DownloadData> ReadAsync();
+        Task ReadAsync(AddDownloadCount addCount);
     }
-}
+}

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -73,6 +73,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj" />
+    <ProjectReference Include="..\NuGet.Services.Configuration\NuGet.Services.Configuration.csproj" />
     <ProjectReference Include="..\NuGet.Services.Logging\NuGet.Services.Logging.csproj" />
     <ProjectReference Include="..\NuGet.Services.Sql\NuGet.Services.Sql.csproj" />
     <ProjectReference Include="..\NuGetGallery.Core\NuGetGallery.Core.csproj" />

src/Catalog/ServiceCollectionExtensions.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Azure.Storage.Blobs;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Configuration;
+using NuGet.Services.Metadata.Catalog;
+
+namespace Microsoft.Extensions.DependencyInjection
+{
+    public static class ServiceCollectionExtensions
+    {
+        public static IServiceCollection AddDownloadsV1JsonClient(this IServiceCollection services, Func<IServiceProvider, string> urlFactory)
+        {
+            services.AddSingleton<IDownloadsV1JsonClient>(provider =>
+            {
+                var url = urlFactory(provider);
+
+                var configuration = provider.GetRequiredService<IConfiguration>();
+                var blobClient = new BlobClient(new Uri(url), configuration.GetTokenCredential());
+
+                var logger = provider.GetRequiredService<ILogger<DownloadsV1JsonClient>>();
+
+                return new DownloadsV1JsonClient(blobClient, logger);
+            });
+            return services;
+        }
+    }
+}

src/NuGet.Jobs.Auxiliary2AzureSearch/Job.cs

@@ -1,8 +1,9 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using NuGet.Services.AzureSearch;
 using NuGet.Services.AzureSearch.Auxiliary2AzureSearch;
 
@@ -19,6 +20,11 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
             services.Configure<Auxiliary2AzureSearchConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
             services.Configure<AzureSearchJobConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
             services.Configure<AzureSearchConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
+            services.AddDownloadsV1JsonClient(provider =>
+            {
+                var jsonConfigurationAccessor = provider.GetRequiredService<IOptionsSnapshot<Auxiliary2AzureSearchConfiguration>>();
+                return jsonConfigurationAccessor.Value.DownloadsV1JsonUrl;
+            });
         }
     }
 }

src/NuGet.Jobs.Db2AzureSearch/Job.cs

@@ -1,8 +1,9 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using NuGet.Services.AzureSearch;
 using NuGet.Services.AzureSearch.AuxiliaryFiles;
 using NuGet.Services.AzureSearch.Db2AzureSearch;
@@ -26,6 +27,11 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
                 configurationRoot.GetSection(DevelopmentConfigurationSectionName));
             services.Configure<Db2AzureSearchDevelopmentConfiguration>(
                 configurationRoot.GetSection(DevelopmentConfigurationSectionName));
+            services.AddDownloadsV1JsonClient(provider =>
+            {
+                var jsonConfigurationAccessor = provider.GetRequiredService<IOptionsSnapshot<Db2AzureSearchConfiguration>>();
+                return jsonConfigurationAccessor.Value.DownloadsV1JsonUrl;
+            });
         }
     }
 }

src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateDownloadsCommand.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -117,7 +117,7 @@ private async Task<bool> PushIndexChangesAsync()
 
             // The "new" data in this case is from the statistics pipeline.
             _logger.LogInformation("Fetching new download count data by URL.");
-            var newData = await _downloadsV1JsonClient.ReadAsync(_options.Value.DownloadsV1JsonUrl);
+            var newData = await _downloadsV1JsonClient.ReadAsync();
 
             _logger.LogInformation("Removing invalid IDs and versions from the old downloads data.");
             CleanDownloadData(oldResult.Data);
@@ -436,4 +436,4 @@ private void CleanDownloadData(DownloadData data)
                 nonNormalizedVersionCount);
         }
     }
-}
+}

src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistrationFromDbProducer.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -94,7 +94,7 @@ public async Task<InitialAuxiliaryData> ProduceWorkAsync(
             // Fetch the download data from the auxiliary file, since this is what is used for displaying download
             // counts in the search service. We don't use the gallery DB values as they are different from the
             // auxiliary file.
-            var downloads = await _downloadsV1JsonClient.ReadAsync(_options.Value.DownloadsV1JsonUrl);
+            var downloads = await _downloadsV1JsonClient.ReadAsync();
             var popularityTransfers = await GetPopularityTransfersAsync();
 
             // Apply changes from popularity transfers.
@@ -442,4 +442,4 @@ public PackageRegistrationInfo(int key, string id, string[] owners, bool isVerif
             public bool IsVerified { get; }
         }
     }
-}
+}

src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -26,7 +26,6 @@
 using NuGet.Services.AzureSearch.Db2AzureSearch;
 using NuGet.Services.AzureSearch.SearchService;
 using NuGet.Services.AzureSearch.Wrappers;
-using NuGet.Services.Metadata.Catalog;
 using NuGet.Services.Metadata.Catalog.Persistence;
 using NuGet.Services.V3;
 using NuGetGallery;
@@ -316,7 +315,6 @@ public static IServiceCollection AddAzureSearch(
                     }
                 });
 
-            services.AddTransient<IDownloadsV1JsonClient, DownloadsV1JsonClient>();
             services.AddSingleton<IAuxiliaryDataCache, AuxiliaryDataCache>();
             services.AddScoped(p => p.GetRequiredService<IAuxiliaryDataCache>().Get());
             services.AddSingleton<IAuxiliaryFileReloader, AuxiliaryFileReloader>();

src/NuGet.Services.Configuration/ConfigurationExtensions.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved. 
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 
+
+using Azure.Core;
+using Azure.Identity;
+using Microsoft.Extensions.Configuration;
+
+namespace NuGet.Services.Configuration
+{
+    public static class ConfigurationExtensions
+    {
+        public static TokenCredential GetTokenCredential(this IConfiguration configuration)
+        {
+            string clientId = configuration[Constants.ManagedIdentityClientIdKey];
+
+            if (!string.IsNullOrWhiteSpace(clientId))
+            {
+                return new ManagedIdentityCredential(clientId);
+            }
+
+            return new DefaultAzureCredential();
+        }
+    }
+}

src/NuGetGallery.Core/CoreConstants.cs

@@ -33,15 +33,13 @@ public static class Folders
         {
             public const string UserCertificatesFolderName = "user-certificates";
             public const string ContentFolderName = "content";
-            public const string DownloadsFolderName = "downloads";
             public const string PackageBackupsFolderName = "package-backups";
             public const string PackageReadMesFolderName = "readmes";
             public const string PackagesFolderName = "packages";
             public const string PackagesContentFolderName = "packages-content";
             public const string UploadsFolderName = "uploads";
             public const string ValidationFolderName = "validation";
             public const string RevalidationFolderName = "revalidation";
-            public const string StatusFolderName = "status";
             public const string SymbolPackagesFolderName = "symbol-packages";
             public const string SymbolPackageBackupsFolderName = "symbol-package-backups";
             public const string FlatContainerFolderName = "v3-flatcontainer";
@@ -57,4 +55,4 @@ public static class Folders
 
         public const string LoginDiscontinuationConfigFileName = "Login-Discontinuation-Configuration.json";
     }
-}
+}

src/NuGetGallery.Core/Services/GalleryCloudBlobContainerInformationProvider.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -11,7 +11,6 @@ public class GalleryCloudBlobContainerInformationProvider : ICloudBlobContainerI
     {
         private static readonly HashSet<string> KnownPublicFolders = new HashSet<string> {
             CoreConstants.Folders.PackagesFolderName,
-            CoreConstants.Folders.DownloadsFolderName,
             CoreConstants.Folders.SymbolPackagesFolderName,
             CoreConstants.Folders.FlatContainerFolderName,
         };
@@ -23,7 +22,6 @@ public class GalleryCloudBlobContainerInformationProvider : ICloudBlobContainerI
             CoreConstants.Folders.ValidationFolderName,
             CoreConstants.Folders.UserCertificatesFolderName,
             CoreConstants.Folders.RevalidationFolderName,
-            CoreConstants.Folders.StatusFolderName,
             CoreConstants.Folders.PackagesContentFolderName,
             CoreConstants.Folders.PackageBackupsFolderName,
             CoreConstants.Folders.SymbolPackageBackupsFolderName,
@@ -41,11 +39,9 @@ public string GetCacheControl(string folderName)
                 case CoreConstants.Folders.PackageBackupsFolderName:
                 case CoreConstants.Folders.UploadsFolderName:
                 case CoreConstants.Folders.SymbolPackageBackupsFolderName:
-                case CoreConstants.Folders.DownloadsFolderName:
                 case CoreConstants.Folders.PackageReadMesFolderName:
                 case CoreConstants.Folders.ContentFolderName:
                 case CoreConstants.Folders.RevalidationFolderName:
-                case CoreConstants.Folders.StatusFolderName:
                 case CoreConstants.Folders.UserCertificatesFolderName:
                 case CoreConstants.Folders.PackagesContentFolderName:
                 case CoreConstants.Folders.FlatContainerFolderName:
@@ -70,15 +66,11 @@ public string GetContentType(string folderName)
                 case CoreConstants.Folders.FlatContainerFolderName:
                     return CoreConstants.PackageContentType;
 
-                case CoreConstants.Folders.DownloadsFolderName:
-                    return CoreConstants.OctetStreamContentType;
-
                 case CoreConstants.Folders.PackageReadMesFolderName:
                     return CoreConstants.TextContentType;
 
                 case CoreConstants.Folders.ContentFolderName:
                 case CoreConstants.Folders.RevalidationFolderName:
-                case CoreConstants.Folders.StatusFolderName:
                     return CoreConstants.JsonContentType;
 
                 case CoreConstants.Folders.UserCertificatesFolderName:

src/NuGetGallery.Services/Configuration/IBlobStorageConfiguration.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 namespace NuGetGallery.Services
@@ -7,5 +7,7 @@ public interface IBlobStorageConfiguration
     {
         string ConnectionString { get; }
         bool ReadAccessGeoRedundant { get; }
+        string MsiClientId { get; }
+        bool UseMsi { get; }
     }
 }