Merge pull request #10279 from NuGet/dev

[ReleasePrep][2024.11.22]RI of dev into main

src/AccountDeleter/Configuration/AccountDeleteConfiguration.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System.Collections.Generic;
@@ -27,6 +27,7 @@ public class AccountDeleteConfiguration
         /// </summary>
         public Dictionary<string, string> TemplateReplacements { get; set; }
 
+
         /// <summary>
         /// Storage container connection string where gallery content can be found
         /// </summary>
@@ -50,4 +51,4 @@ public SourceConfiguration GetSourceConfiguration(string source)
             throw new UnknownSourceException();
         }
     }
-}
+}

src/AccountDeleter/EmptyFeatureFlagService.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using NuGet.Services.Entities;
@@ -323,5 +323,10 @@ public bool IsAdvancedFrameworkFilteringEnabled(User user)
         {
             throw new NotImplementedException();
         }
+
+        public bool CanUseFederatedCredentials(User user)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

src/AccountDeleter/Job.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -27,6 +27,7 @@
 using NuGetGallery.Features;
 using NuGetGallery.Infrastructure.Authentication;
 using NuGetGallery.Security;
+using ConfigConstants = NuGet.Services.Configuration.Constants;
 
 namespace NuGetGallery.AccountDeleter
 {
@@ -104,6 +105,13 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
             services.AddScoped<ITelemetryClient, TelemetryClientWrapper>(
                 sp => TelemetryClientWrapper.UseTelemetryConfiguration(ApplicationInsightsConfiguration.TelemetryConfiguration));
 
+            services.AddScoped<ICloudBlobClient>(serviceProvider =>
+            {
+                var options = serviceProvider.GetRequiredService<IOptionsSnapshot<AccountDeleteConfiguration>>();
+                return CloudBlobClientWrapper.UsingMsi(options.Value.GalleryStorageConnectionString,
+                    configurationRoot[ConfigConstants.StorageManagedIdentityClientIdPropertyName]);
+            });
+
             ConfigureGalleryServices(services);
         }
 
@@ -160,14 +168,6 @@ protected void ConfigureGalleryServices(IServiceCollection services)
                     return new SupportRequestDbContext(connection);
                 });
 
-                services.AddScoped<ICloudBlobClient>(sp =>
-                {
-                    var options = sp.GetRequiredService<IOptionsSnapshot<AccountDeleteConfiguration>>();
-                    var optionsSnapshot = options.Value;
-
-                    return new CloudBlobClientWrapper(optionsSnapshot.GalleryStorageConnectionString, readAccessGeoRedundant: true);
-                });
-
                 services.AddScoped<ITelemetryService, TelemetryService>();
                 services.AddScoped<ISecurityPolicyService, SecurityPolicyService>();
                 services.AddScoped<IAppConfiguration, GalleryConfiguration>();

src/Catalog/Dnx/DnxCatalogCollector.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -193,7 +193,7 @@ await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async ca
             return processedCatalogEntries;
         }
 
-        private async Task<bool> AreRequiredPropertiesPresentAsync(Storage destinationStorage, Uri destinationUri)
+        private async Task<bool> AreRequiredPropertiesPresentAsync(Persistence.Storage destinationStorage, Uri destinationUri)
         {
             var azureStorage = destinationStorage as IAzureStorage;
 
@@ -561,4 +561,4 @@ internal static CatalogEntry Create(CatalogCommitItem item)
             }
         }
     }
-}
+}

src/Catalog/Dnx/DnxMaker.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -16,6 +16,7 @@
 using NuGet.Versioning;
 
 using ILogger = Microsoft.Extensions.Logging.ILogger;
+using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage;
 
 namespace NuGet.Services.Metadata.Catalog.Dnx
 {
@@ -154,7 +155,7 @@ public async Task DeletePackageAsync(string id, string version, CancellationToke
             await DeleteNupkgAsync(storage, id, normalizedVersion, cancellationToken);
         }
 
-        public async Task<bool> HasPackageInIndexAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+        public async Task<bool> HasPackageInIndexAsync(CatalogStorage storage, string id, string version, CancellationToken cancellationToken)
         {
             if (storage == null)
             {
@@ -179,7 +180,7 @@ public async Task<bool> HasPackageInIndexAsync(Storage storage, string id, strin
             return versionsContext.Versions.Contains(parsedVersion);
         }
 
-        private async Task<Uri> SaveNuspecAsync(Storage storage, string id, string version, string nuspec, CancellationToken cancellationToken)
+        private async Task<Uri> SaveNuspecAsync(CatalogStorage storage, string id, string version, string nuspec, CancellationToken cancellationToken)
         {
             var relativeAddress = GetRelativeAddressNuspec(id, version);
             var nuspecUri = new Uri(storage.BaseAddress, relativeAddress);
@@ -230,7 +231,7 @@ public async Task UpdatePackageVersionIndexAsync(string id, Action<HashSet<NuGet
             }
         }
 
-        private async Task<VersionsResult> GetVersionsAsync(Storage storage, CancellationToken cancellationToken)
+        private async Task<VersionsResult> GetVersionsAsync(CatalogStorage storage, CancellationToken cancellationToken)
         {
             var relativeAddress = "index.json";
             var resourceUri = new Uri(storage.BaseAddress, relativeAddress);
@@ -265,7 +266,7 @@ private StorageContent CreateContent(IEnumerable<string> versions)
             return new StringStorageContent(obj.ToString(), "application/json", Constants.NoStoreCacheControl);
         }
 
-        private async Task<Uri> SaveNupkgAsync(Stream nupkgStream, Storage storage, string id, string version, CancellationToken cancellationToken)
+        private async Task<Uri> SaveNupkgAsync(Stream nupkgStream, CatalogStorage storage, string id, string version, CancellationToken cancellationToken)
         {
             Uri nupkgUri = new Uri(storage.BaseAddress, GetRelativeAddressNupkg(id, version));
             var content = new StreamStorageContent(
@@ -280,7 +281,7 @@ private async Task<Uri> SaveNupkgAsync(Stream nupkgStream, Storage storage, stri
 
         private async Task<Uri> CopyNupkgAsync(
             IStorage sourceStorage,
-            Storage destinationStorage,
+            CatalogStorage destinationStorage,
             string id, string version, CancellationToken cancellationToken)
         {
             var packageFileName = PackageUtility.GetPackageFileName(id, version);
@@ -300,7 +301,7 @@ await sourceStorage.CopyAsync(
 
         private async Task CopyIconFromAzureStorageIfExistAsync(
             IAzureStorage sourceStorage,
-            Storage destinationStorage,
+            CatalogStorage destinationStorage,
             string packageId,
             string normalizedPackageVersion,
             string iconFilename,
@@ -321,7 +322,7 @@ await CopyIconAsync(
         private async Task CopyIconFromNupkgStreamAsync(
             Stream nupkgStream,
             string iconFilename,
-            Storage destinationStorage,
+            CatalogStorage destinationStorage,
             string packageId,
             string normalizedPackageVersion,
             CancellationToken cancellationToken)
@@ -338,7 +339,7 @@ await CopyIconAsync(
         private async Task CopyIconAsync(
             Stream packageStream,
             string iconFilename,
-            Storage destinationStorage,
+            CatalogStorage destinationStorage,
             string packageId,
             string normalizedPackageVersion,
             CancellationToken cancellationToken)
@@ -366,7 +367,7 @@ await ExtractAndStoreIconAsync(
         private async Task ExtractAndStoreIconAsync(
             Stream packageStream,
             string iconPath,
-            Storage destinationStorage,
+            CatalogStorage destinationStorage,
             Uri destinationUri,
             CancellationToken cancellationToken,
             string packageId,
@@ -406,7 +407,7 @@ private async Task<Stream> GetPackageStreamAsync(
             return await packageSourceBlob.GetStreamAsync(cancellationToken);
         }
 
-        private async Task DeleteNuspecAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+        private async Task DeleteNuspecAsync(CatalogStorage storage, string id, string version, CancellationToken cancellationToken)
         {
             string relativeAddress = GetRelativeAddressNuspec(id, version);
             Uri nuspecUri = new Uri(storage.BaseAddress, relativeAddress);
@@ -416,7 +417,7 @@ private async Task DeleteNuspecAsync(Storage storage, string id, string version,
             }
         }
 
-        private async Task DeleteNupkgAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+        private async Task DeleteNupkgAsync(CatalogStorage storage, string id, string version, CancellationToken cancellationToken)
         {
             string relativeAddress = GetRelativeAddressNupkg(id, version);
             Uri nupkgUri = new Uri(storage.BaseAddress, relativeAddress);
@@ -426,7 +427,7 @@ private async Task DeleteNupkgAsync(Storage storage, string id, string version,
             }
         }
 
-        private async Task DeleteIconAsync(Storage storage, string id, string version, CancellationToken cancellationToken)
+        private async Task DeleteIconAsync(CatalogStorage storage, string id, string version, CancellationToken cancellationToken)
         {
             string relativeAddress = GetRelativeAddressIcon(id, version);
             Uri iconUri = new Uri(storage.BaseAddress, relativeAddress);
@@ -479,4 +480,4 @@ public VersionsResult(string relativeAddress, Uri resourceUri, HashSet<NuGetVers
             public HashSet<NuGetVersion> Versions { get; }
         }
     }
-}
+}

src/Catalog/DurableCursor.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -12,10 +12,10 @@ namespace NuGet.Services.Metadata.Catalog
     public class DurableCursor : ReadWriteCursor
     {
         Uri _address;
-        Storage _storage;
+        Persistence.Storage _storage;
         DateTime _defaultValue;
 
-        public DurableCursor(Uri address, Storage storage, DateTime defaultValue)
+        public DurableCursor(Uri address, Persistence.Storage storage, DateTime defaultValue)
         {
             _address = address;
             _storage = storage;
@@ -43,4 +43,4 @@ public override async Task LoadAsync(CancellationToken cancellationToken)
             Value = obj["value"].ToObject<DateTime>();
         }
     }
-}
+}

src/Catalog/Helpers/DeletionAuditEntry.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -163,7 +163,7 @@ public static Task<IEnumerable<DeletionAuditEntry>> GetAsync(
             DateTime? maxTime = null,
             ILogger logger = null)
         {
-            Storage storage = auditingStorageFactory.Create(package != null ? GetAuditRecordPrefixFromPackage(package) : null);
+            Persistence.Storage storage = auditingStorageFactory.Create(package != null ? GetAuditRecordPrefixFromPackage(package) : null);
             return GetAsync(storage, cancellationToken, minTime, maxTime, logger);
         }
 
@@ -258,4 +258,4 @@ private static bool IsPackageDelete(StorageListItem auditRecord)
             return FileNameSuffixes.Any(suffix => fileName.EndsWith(suffix));
         }
     }
-}
+}

src/Catalog/Icons/CatalogLeafDataProcessor.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -12,6 +12,8 @@
 using NuGet.Services.Metadata.Catalog.Helpers;
 using NuGet.Services.Metadata.Catalog.Persistence;
 
+using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage;
+
 namespace NuGet.Services.Metadata.Catalog.Icons
 {
     public class CatalogLeafDataProcessor : ICatalogLeafDataProcessor
@@ -42,7 +44,7 @@ public CatalogLeafDataProcessor(
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task ProcessPackageDeleteLeafAsync(Storage storage, CatalogCommitItem item, CancellationToken cancellationToken)
+        public async Task ProcessPackageDeleteLeafAsync(CatalogStorage storage, CatalogCommitItem item, CancellationToken cancellationToken)
         {
             var targetStoragePath = GetTargetStorageIconPath(item);
             await _iconProcessor.DeleteIconAsync(storage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString());

src/Catalog/Icons/ICatalogLeafDataProcessor.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System.Threading;
@@ -9,7 +9,7 @@ namespace NuGet.Services.Metadata.Catalog.Icons
 {
     public interface ICatalogLeafDataProcessor
     {
-        Task ProcessPackageDeleteLeafAsync(Storage storage, CatalogCommitItem item, CancellationToken cancellationToken);
+        Task ProcessPackageDeleteLeafAsync(Persistence.Storage storage, CatalogCommitItem item, CancellationToken cancellationToken);
         Task ProcessPackageDetailsLeafAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, string iconUrlString, string iconFile, CancellationToken cancellationToken);
     }
-}
+}

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -76,6 +76,7 @@
     <ProjectReference Include="..\NuGet.Services.Configuration\NuGet.Services.Configuration.csproj" />
     <ProjectReference Include="..\NuGet.Services.Logging\NuGet.Services.Logging.csproj" />
     <ProjectReference Include="..\NuGet.Services.Sql\NuGet.Services.Sql.csproj" />
+    <ProjectReference Include="..\NuGet.Services.Storage\NuGet.Services.Storage.csproj" />
     <ProjectReference Include="..\NuGetGallery.Core\NuGetGallery.Core.csproj" />
   </ItemGroup>
 

src/Catalog/Persistence/AzureStorage.cs

@@ -11,11 +11,13 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Azure;
+using Azure.Identity;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
 using NuGet.Protocol;
 using NuGet.Services.Metadata.Catalog.Extensions;
+using NuGet.Services.Storage;
 using NuGetGallery;
 
 namespace NuGet.Services.Metadata.Catalog.Persistence
@@ -33,7 +35,7 @@ public class AzureStorage : Storage, IAzureStorage
         public static readonly TimeSpan DefaultMaxExecutionTime = TimeSpan.FromMinutes(10);
 
         public AzureStorage(
-            BlobServiceClient blobServiceClient,
+            IBlobServiceClientFactory blobServiceClient,
             string containerName,
             string path,
             Uri baseAddress,
@@ -92,7 +94,7 @@ private static ICloudBlobDirectory GetCloudBlobDirectoryUri(Uri storageBaseUri)
 
             var blobEndpoint = new Uri(storageBaseUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped));
             // Create BlobServiceClient with anonymous credentials
-            var blobServiceClient = new BlobServiceClient(blobEndpoint);
+            var blobServiceClient = new BlobServiceClientFactory(blobEndpoint);
 
             string containerName = pathSegments[0];
             string pathInContainer = string.Join("/", pathSegments.Skip(1));

src/Catalog/Persistence/AzureStorageFactory.cs

@@ -1,15 +1,16 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
 using Azure.Storage.Blobs;
 using NuGet.Protocol;
+using NuGet.Services.Storage;
 
 namespace NuGet.Services.Metadata.Catalog.Persistence
 {
     public class AzureStorageFactory : StorageFactory
     {
-        private readonly BlobServiceClient _blobServiceClient;
+        private readonly IBlobServiceClientFactory _blobServiceClient;
         private readonly string _containerName;
         private readonly string _path;
         private readonly Uri _differentBaseAddress = null;
@@ -19,7 +20,7 @@ public class AzureStorageFactory : StorageFactory
         private readonly bool _initializeContainer;
 
         public AzureStorageFactory(
-            BlobServiceClient blobServiceClient,
+            IBlobServiceClientFactory blobServiceClient,
             string containerName,
             TimeSpan maxExecutionTime,
             TimeSpan serverTimeout,
@@ -107,4 +108,4 @@ public override Storage Create(string name = null)
                 Throttle);
         }
     }
-}
+}