Add support for NuGet V2 upstream package sources (loic-sharma#630) (#30

)

To configure the mirror service to use the NuGet V2 client:

```json
"Mirror": {
  "Enabled": true,
  "PackageSource": "https://www.nuget.org/api/v2/",
  "Legacy": true
},
```

Co-authored-by: Patrik Svensson <patriksvensson@users.noreply.github.com>

src/BaGet.Core/BaGet.Core.csproj

@@ -12,7 +12,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="$(MicrosoftExtensionsPackageVersion)" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionsPackageVersion)" />
     <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(MicrosoftExtensionsPackageVersion)" />
-    <PackageReference Include="NuGet.Packaging" Version="$(NuGetPackageVersion)" />
+    <PackageReference Include="NuGet.Protocol" Version="$(NuGetPackageVersion)" />
     <PackageReference Include="System.Reflection.Metadata" Version="1.6.0" />
   </ItemGroup>
 

src/BaGet.Core/Configuration/MirrorOptions.cs

@@ -17,6 +17,11 @@ public class MirrorOptions : IValidatableObject
         /// </summary>
         public Uri PackageSource { get; set; }
 
+        /// <summary>
+        /// Whether or not the package source is a v2 package source feed.
+        /// </summary>
+        public bool Legacy { get; set; }
+
         /// <summary>
         /// The time before a download from the package source times out.
         /// </summary>

src/BaGet.Core/Extensions/DependencyInjectionExtensions.cs

@@ -99,11 +99,14 @@ private static void AddBaGetServices(this IServiceCollection services)
             services.TryAddTransient<DatabaseSearchService>();
             services.TryAddTransient<FileStorageService>();
             services.TryAddTransient<MirrorService>();
+            services.TryAddTransient<MirrorV2Client>();
+            services.TryAddTransient<MirrorV3Client>();
             services.TryAddTransient<NullMirrorService>();
             services.TryAddSingleton<NullStorageService>();
             services.TryAddTransient<PackageService>();
 
             services.TryAddTransient(IMirrorServiceFactory);
+            services.TryAddTransient(IMirrorNuGetClientFactory);
         }
 
         private static void AddDefaultProviders(this IServiceCollection services)
@@ -195,8 +198,16 @@ private static IMirrorService IMirrorServiceFactory(IServiceProvider provider)
         {
             var options = provider.GetRequiredService<IOptionsSnapshot<MirrorOptions>>();
             var service = options.Value.Enabled ? typeof(MirrorService) : typeof(NullMirrorService);
-
+            
             return (IMirrorService)provider.GetRequiredService(service);
         }
+
+        private static IMirrorNuGetClient IMirrorNuGetClientFactory(IServiceProvider provider)
+        {
+            var options = provider.GetRequiredService<IOptionsSnapshot<MirrorOptions>>();
+            var service = options.Value.Legacy ? typeof(MirrorV2Client) : typeof(MirrorV3Client);
+
+            return (IMirrorNuGetClient)provider.GetRequiredService(service);
+        }
     }
 }

src/BaGet.Core/Mirror/Clients/MirrorV2Client.cs

@@ -0,0 +1,114 @@
+using BaGet.Protocol.Models;
+using Microsoft.Extensions.Options;
+using NuGet.Common;
+using NuGet.Configuration;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BaGet.Core
+{
+    internal sealed class MirrorV2Client : IMirrorNuGetClient
+    {
+        private readonly ILogger _logger;
+        private readonly SourceCacheContext _cache;
+        private readonly SourceRepository _repository;
+
+        public MirrorV2Client(IOptionsSnapshot<MirrorOptions> options)
+        {
+            if (options is null)
+            {
+                throw new ArgumentNullException(nameof(options));
+            }
+
+            if (options.Value?.PackageSource?.AbsolutePath == null)
+            {
+                throw new ArgumentException("No mirror package source has been set.");
+            }
+
+            _logger = NullLogger.Instance;
+            _cache = new SourceCacheContext();
+            _repository = Repository.Factory.GetCoreV2(new PackageSource(options.Value.PackageSource.AbsoluteUri));
+        }
+
+        public async Task<IReadOnlyList<NuGetVersion>> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken)
+        {
+            var resource = await _repository.GetResourceAsync<FindPackageByIdResource>();
+            var versions = await resource.GetAllVersionsAsync(id, _cache, _logger, cancellationToken);
+
+            return versions.ToList();
+        }
+
+        public async Task<IReadOnlyList<PackageMetadata>> GetPackageMetadataAsync(string id, CancellationToken cancellationToken)
+        {
+            var resource = await _repository.GetResourceAsync<PackageMetadataResource>();
+            var packages = await resource.GetMetadataAsync(id, includePrerelease: true, includeUnlisted: false, _cache, _logger, cancellationToken);
+
+            var result = new List<PackageMetadata>();
+            foreach (var package in packages)
+            {
+                result.Add(new PackageMetadata
+                {
+                    Authors = package.Authors,
+                    Description = package.Description,
+                    IconUrl = package.IconUrl?.AbsoluteUri,
+                    LicenseUrl = package.LicenseUrl?.AbsoluteUri,
+                    Listed = package.IsListed,
+                    PackageId = id,
+                    Summary = package.Summary,
+                    Version = package.Identity.Version.ToString(),
+                    Tags = package.Tags?.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries),
+                    Title = package.Title,
+                    RequireLicenseAcceptance = package.RequireLicenseAcceptance,
+                    Published = package.Published?.UtcDateTime ?? DateTimeOffset.MinValue,
+                    ProjectUrl = package.ProjectUrl?.AbsoluteUri,
+                    DependencyGroups = GetDependencies(package),
+                });
+            }
+
+            return result;
+        }
+
+        public async Task<Stream> DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken)
+        {
+            var packageStream = new MemoryStream();
+            var resource = await _repository.GetResourceAsync<FindPackageByIdResource>();
+            await resource.CopyNupkgToStreamAsync(id, version, packageStream, _cache, _logger, cancellationToken);
+            packageStream.Seek(0, SeekOrigin.Begin);
+
+            return packageStream;
+        }
+
+        private IReadOnlyList<DependencyGroupItem> GetDependencies(IPackageSearchMetadata package)
+        {
+            var groupItems = new List<DependencyGroupItem>();
+            foreach (var set in package.DependencySets)
+            {
+                var item = new DependencyGroupItem
+                {
+                    TargetFramework = set.TargetFramework.Framework,
+                    Dependencies = new List<DependencyItem>()
+                };
+
+                foreach (var dependency in set.Packages)
+                {
+                    item.Dependencies.Add(new DependencyItem
+                    {
+                        Id = dependency.Id,
+                        Range = dependency.VersionRange.ToNormalizedString(),
+                    });
+                }
+
+                groupItems.Add(item);
+            }
+
+            return groupItems;
+        }
+    }
+}

src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs

@@ -0,0 +1,36 @@
+using BaGet.Protocol;
+using BaGet.Protocol.Models;
+using NuGet.Versioning;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BaGet.Core
+{
+    internal sealed class MirrorV3Client : IMirrorNuGetClient
+    {
+        private readonly NuGetClient _client;
+
+        public MirrorV3Client(NuGetClient client)
+        {
+            _client = client ?? throw new ArgumentNullException(nameof(client));
+        }
+
+        public async Task<Stream> DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken)
+        {
+            return await _client.DownloadPackageAsync(id, version, cancellationToken);
+        }
+
+        public async Task<IReadOnlyList<PackageMetadata>> GetPackageMetadataAsync(string id, CancellationToken cancellationToken)
+        {
+            return await _client.GetPackageMetadataAsync(id, cancellationToken);
+        }
+
+        public async Task<IReadOnlyList<NuGetVersion>> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken)
+        {
+            return await _client.ListPackageVersionsAsync(id, includeUnlisted, cancellationToken);
+        }
+    }
+}

src/BaGet.Core/Mirror/IMirrorNuGetClient.cs

@@ -0,0 +1,16 @@
+using BaGet.Protocol.Models;
+using NuGet.Versioning;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BaGet.Core
+{
+    public interface IMirrorNuGetClient
+    {
+        Task<IReadOnlyList<NuGetVersion>> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken);
+        Task<IReadOnlyList<PackageMetadata>> GetPackageMetadataAsync(string id, CancellationToken cancellationToken);
+        Task<Stream> DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
+    }
+}

src/BaGet.Core/Mirror/MirrorService.cs

@@ -16,13 +16,13 @@ namespace BaGet.Core
     public class MirrorService : IMirrorService
     {
         private readonly IPackageService _localPackages;
-        private readonly NuGetClient _upstreamClient;
+        private readonly IMirrorNuGetClient _upstreamClient;
         private readonly IPackageIndexingService _indexer;
         private readonly ILogger<MirrorService> _logger;
 
         public MirrorService(
             IPackageService localPackages,
-            NuGetClient upstreamClient,
+            IMirrorNuGetClient upstreamClient,
             IPackageIndexingService indexer,
             ILogger<MirrorService> logger)
         {
@@ -32,6 +32,19 @@ public MirrorService(
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
+        public static MirrorService Create(
+            IPackageService localPackages,
+            NuGetClient client,
+            IPackageIndexingService indexer,
+            ILogger<MirrorService> logger)
+        {
+            return new MirrorService(
+                localPackages,
+                new MirrorV3Client(client),
+                indexer,
+                logger);
+        }
+
         public async Task<IReadOnlyList<NuGetVersion>> FindPackageVersionsOrNullAsync(
             string id,
             CancellationToken cancellationToken)

src/BaGet/appsettings.json

@@ -19,6 +19,8 @@
 
   "Mirror": {
     "Enabled": false,
+    // Uncomment this to use the NuGet v2 protocol
+    //"Legacy": true,
     "PackageSource": "https://api.nuget.org/v3/index.json"
   },
 

tests/BaGet.Core.Tests/Mirror/MirrorServiceTests.cs

@@ -61,7 +61,7 @@ public FactsBase()
                 _upstream = new Mock<NuGetClient>();
                 _indexer = new Mock<IPackageIndexingService>();
 
-                _target = new MirrorService(
+                _target = MirrorService.Create(
                     _packages.Object,
                     _upstream.Object,
                     _indexer.Object,