improve nuget package detection with SDK-managed packages

.gitmodules

@@ -2,3 +2,6 @@
 	path = nuget/helpers/lib/NuGet.Client
 	url = https://github.com/NuGet/NuGet.Client
 	branch = release-6.12.x
+[submodule "nuget/helpers/lib/dotnet-core"]
+	path = nuget/helpers/lib/dotnet-core
+	url = https://github.com/dotnet/core

nuget/helpers/lib/NuGetUpdater/.gitignore

@@ -4,4 +4,5 @@ bin/
 obj/
 Properties/launchSettings.json
 NuGetUpdater.sln.DotSettings.user
+NuGetUpdater.Core/dotnet-package-correlation.json
 *.binlog

nuget/helpers/lib/NuGetUpdater/Directory.Packages.props

@@ -26,6 +26,7 @@
     <PackageVersion Include="MSBuild.StructuredLogger" Version="2.2.386" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="NuGet.Core" Version="2.14.0" Aliases="CoreV2" />
+    <PackageVersion Include="Semver" Version="3.0.0" />
     <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
     <PackageVersion Include="System.ComponentModel.Composition" Version="9.0.0" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />

nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(CommonTargetFramework)</TargetFramework>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.CommandLine" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DotNetPackageCorrelation\DotNetPackageCorrelation.csproj" />
+  </ItemGroup>
+
+</Project>

nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs

@@ -0,0 +1,32 @@
+using System.CommandLine;
+using System.Text.Json;
+
+using DotNetPackageCorrelation;
+
+namespace DotNetPackageCorrelation.Cli;
+
+public class Program
+{
+    public static async Task<int> Main(string[] args)
+    {
+        var coreLocationOption = new Option<DirectoryInfo>("--core-location", "The location of the .NET Core source code.") { IsRequired = true };
+        var outputOption = new Option<FileInfo>("--output", "The location to write the result.") { IsRequired = true };
+        var command = new Command("build")
+        {
+            coreLocationOption,
+            outputOption,
+        };
+        command.TreatUnmatchedTokensAsErrors = true;
+        command.SetHandler(async (coreLocationDirectory, output) =>
+        {
+            // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory
+            var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes"));
+            var correlator = new Correlator(releaseNotesDirectory);
+            var (sdkPackages, _warnings) = await correlator.RunAsync();
+            var json = JsonSerializer.Serialize(sdkPackages, Correlator.SerializerOptions);
+            await File.WriteAllTextAsync(output.FullName, json);
+        }, coreLocationOption, outputOption);
+        var exitCode = await command.InvokeAsync(args);
+        return exitCode;
+    }
+}

nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs

@@ -0,0 +1,92 @@
+using Semver;
+
+using Xunit;
+
+namespace DotNetPackageCorrelation.Tests;
+
+public class CorrelatorTests
+{
+    [Fact]
+    public async Task FileHandling_AllFilesShapedAppropriately()
+    {
+        // the JSON and markdown are shaped as expected
+        var (sdkPackages, warnings) = await RunFromFilesAsync(
+            ("8.0/releases.json", """
+                {
+                    "releases": [
+                        {
+                            "sdk": {
+                                "version": "8.0.100",
+                                "runtime-version": "8.0.0"
+                            }
+                        }
+                    ]
+                }
+                """),
+            ("8.0/8.0.0/8.0.0.md", """
+                Package name | Version
+                :-- | :--
+                Package.A | 8.0.0
+                Package.B | 1.2.3
+                """)
+        );
+        Assert.Empty(warnings);
+        AssertPackageVersion(sdkPackages, "8.0.100", "Package.A", "8.0.0");
+        AssertPackageVersion(sdkPackages, "8.0.100", "Package.B", "1.2.3");
+    }
+
+    [Theory]
+    [InlineData("Some.Package | 1.2.3", "Some.Package", "1.2.3")] // happy path
+    [InlineData("Some.Package.1.2.3", "Some.Package", "1.2.3")] // looks like a restore directory
+    [InlineData("Some.Package | 1.2 | 1.2.3.nupkg", "Some.Package", "1.2.3")] // extra columns from a bad filename split
+    [InlineData("Some.Package | 1.2.3.nupkg", "Some.Package", "1.2.3")] // version contains package extension
+    [InlineData("Some.Package | 1.2.3.symbols.nupkg", "Some.Package", "1.2.3")] // version contains symbols package extension
+    [InlineData("some.package.1.2.3.nupkg", "some.package", "1.2.3")] // first column is a filename, second column is missing
+    [InlineData("some.package.1.2.3.nupkg |", "some.package", "1.2.3")] // first column is a filename, second column is empty
+    public void PackagesParsedFromMarkdown(string markdownLine, string expectedPackageName, string expectedPackageVersion)
+    {
+        var markdownContent = $"""
+            Package name | Version
+            :-- | :--
+            {markdownLine}
+            """;
+        var warnings = new List<string>();
+        var packages = Correlator.GetPackagesFromMarkdown("test.md", markdownContent, warnings);
+        Assert.Empty(warnings);
+        var actualpackage = Assert.Single(packages);
+        Assert.Equal(expectedPackageName, actualpackage.Name);
+        Assert.Equal(expectedPackageVersion, actualpackage.Version.ToString());
+    }
+
+    private static void AssertPackageVersion(SdkPackages sdkPackages, string sdkVersion, string packageName, string expectedPackageVersion)
+    {
+        Assert.True(sdkPackages.Sdks.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]");
+        Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]");
+        var actualPackageVersion = packageVersion.ToString();
+        Assert.Equal(expectedPackageVersion, actualPackageVersion);
+    }
+
+    private static async Task<(SdkPackages SdkPackages, IEnumerable<string> Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files)
+    {
+        var testDirectory = Path.Combine(Path.GetDirectoryName(typeof(CorrelatorTests).Assembly.Location)!, "test-data", Guid.NewGuid().ToString("D"));
+        Directory.CreateDirectory(testDirectory);
+
+        try
+        {
+            foreach (var (path, content) in files)
+            {
+                var fullPath = Path.Combine(testDirectory, path);
+                Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
+                await File.WriteAllTextAsync(fullPath, content);
+            }
+
+            var correlator = new Correlator(new DirectoryInfo(testDirectory));
+            var result = await correlator.RunAsync();
+            return result;
+        }
+        finally
+        {
+            Directory.Delete(testDirectory, recursive: true);
+        }
+    }
+}

nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(CommonTargetFramework)</TargetFramework>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DotNetPackageCorrelation\DotNetPackageCorrelation.csproj" />
+  </ItemGroup>
+
+</Project>

nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs

@@ -0,0 +1,31 @@
+using System.Runtime.CompilerServices;
+
+using Semver;
+
+using Xunit;
+
+namespace DotNetPackageCorrelation.Tests;
+
+public class EndToEndTests
+{
+    [Fact]
+    public async Task IntegrationTest()
+    {
+        // arrange
+        var thisFileDirectory = Path.GetDirectoryName(GetThisFilePath())!;
+        var dotnetCoreDirectory = Path.Combine(thisFileDirectory, "..", "..", "dotnet-core");
+        var correlator = new Correlator(new DirectoryInfo(Path.Combine(dotnetCoreDirectory, "release-notes")));
+
+        // act
+        var (sdkPackages, _warnings) = await correlator.RunAsync();
+        var sdkVersion = SemVersion.Parse("8.0.307");
+
+        // SDK 8.0.307 has no System.Text.Json, but 8.0.306 provides System.Text.Json 8.0.5
+        var systemTextJsonPackageVersion = sdkPackages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json");
+
+        // assert
+        Assert.Equal("8.0.5", systemTextJsonPackageVersion?.ToString());
+    }
+
+    private static string GetThisFilePath([CallerFilePath] string? path = null) => path ?? throw new ArgumentNullException(nameof(path));
+}