From 0df58e49a8d48197f39ed0ab45e2662fdf6e903f Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 10:57:03 -0700 Subject: [PATCH 01/13] add .NET SDK package correlation projects --- .../lib/NuGetUpdater/Directory.Packages.props | 1 + .../CorrelatorTests.cs | 92 +++++++++ .../DotNetPackageCorrelation.Test.csproj | 18 ++ .../DotNetPackageCorrelation/Correlator.cs | 192 ++++++++++++++++++ .../DotNetPackageCorrelation.csproj | 14 ++ .../Model/PackageSet.cs | 8 + .../DotNetPackageCorrelation/Model/Release.cs | 9 + .../Model/ReleasesFile.cs | 9 + .../DotNetPackageCorrelation/Model/Sdk.cs | 12 ++ .../Model/SdkPackages.cs | 8 + .../Model/SemVerComparer.cs | 14 ++ .../Model/SemVersionConverter.cs | 31 +++ .../DotNetPackageCorrelation/Program.cs | 30 +++ .../helpers/lib/NuGetUpdater/NuGetUpdater.sln | 13 +- nuget/script/ci-test | 1 + 15 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs diff --git a/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props b/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props index 559d83b38a..8447c8dfc1 100644 --- a/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props +++ b/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props @@ -26,6 +26,7 @@ + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs new file mode 100644 index 0000000000..b24dfe7c4b --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -0,0 +1,92 @@ +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation.Tests; + +public class CorrelatorTests +{ + [Fact] + public async Task AllFilesShapedAppropriately() + { + // the JSON and markdown are shaped as expected + var (packages, warnings) = await RunFromFilesAsync( + ("8.0/releases.json", """ + { + "releases": [ + { + "sdk": { + "version": "8.0.100", + "runtime-version": "8.0.0" + } + } + ] + } + """), + ("8.0/8.0.0/8.0.0.md", """ + Package name | Version + :-- | :-- + Package.A | 8.0.0 + Package.B | 1.2.3 + """) + ); + Assert.Empty(warnings); + AssertPackageVersion(packages, "8.0.100", "Package.A", "8.0.0"); + AssertPackageVersion(packages, "8.0.100", "Package.B", "1.2.3"); + } + + [Theory] + [InlineData("Some.Package | 1.2.3", "Some.Package", "1.2.3")] // happy path + [InlineData("Some.Package.1.2.3", "Some.Package", "1.2.3")] // looks like a restore directory + [InlineData("Some.Package | 1.2 | 1.2.3.nupkg", "Some.Package", "1.2.3")] // extra columns from a bad filename split + [InlineData("Some.Package | 1.2.3.nupkg", "Some.Package", "1.2.3")] // version contains package extension + [InlineData("Some.Package | 1.2.3.symbols.nupkg", "Some.Package", "1.2.3")] // version contains symbols package extension + [InlineData("some.package.1.2.3.nupkg", "some.package", "1.2.3")] // first column is a filename, second column is missing + [InlineData("some.package.1.2.3.nupkg |", "some.package", "1.2.3")] // first column is a filename, second column is empty + public void PackagesParsedFromMarkdown(string markdownLine, string expectedPackageName, string expectedPackageVersion) + { + var markdownContent = $""" + Package name | Version + :-- | :-- + {markdownLine} + """; + var warnings = new List(); + var packages = Correlator.GetPackagesFromMarkdown("test.md", markdownContent, warnings); + Assert.Empty(warnings); + var actualpackage = Assert.Single(packages); + Assert.Equal(expectedPackageName, actualpackage.Name); + Assert.Equal(expectedPackageVersion, actualpackage.Version.ToString()); + } + + private static void AssertPackageVersion(SdkPackages packages, string sdkVersion, string packageName, string expectedPackageVersion) + { + Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK verison [{sdkVersion}]"); + Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); + var actualPackageVersion = packageVersion.ToString(); + Assert.Equal(expectedPackageVersion, actualPackageVersion); + } + + private static async Task<(SdkPackages Packages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) + { + var testDirectory = Path.Combine(Path.GetDirectoryName(typeof(CorrelatorTests).Assembly.Location)!, "test-data", Guid.NewGuid().ToString("D")); + Directory.CreateDirectory(testDirectory); + + try + { + foreach (var (path, content) in files) + { + var fullPath = Path.Combine(testDirectory, path); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + await File.WriteAllTextAsync(fullPath, content); + } + + var correlator = new Correlator(new DirectoryInfo(testDirectory)); + var result = await correlator.RunAsync(); + return result; + } + finally + { + Directory.Delete(testDirectory, recursive: true); + } + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj new file mode 100644 index 0000000000..f7791a7c55 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj @@ -0,0 +1,18 @@ + + + + $(CommonTargetFramework) + Exe + + + + + + + + + + + + + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs new file mode 100644 index 0000000000..01c0d131b6 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -0,0 +1,192 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.RegularExpressions; + +using Semver; + +namespace DotNetPackageCorrelation; + +public partial class Correlator +{ + internal static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = true, + Converters = { new SemVersionConverter() }, + }; + + private readonly DirectoryInfo _releaseNotesDirectory; + + public Correlator(DirectoryInfo releaseNotesDirectory) + { + _releaseNotesDirectory = releaseNotesDirectory; + } + + public async Task<(SdkPackages Packages, IEnumerable Warnings)> RunAsync() + { + var runtimeVersions = new List(); + foreach (var directory in Directory.EnumerateDirectories(_releaseNotesDirectory.FullName)) + { + var directoryName = Path.GetFileName(directory); + if (Version.TryParse(directoryName, out var version)) + { + runtimeVersions.Add(version); + } + } + + var sdkPackages = new SdkPackages(); + var warnings = new List(); + foreach (var version in runtimeVersions) + { + var releasesJsonPath = Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), "releases.json"); + if (!File.Exists(releasesJsonPath)) + { + warnings.Add($"Unable to find releases.json file for version {version}"); + continue; + } + + var releasesJson = await File.ReadAllTextAsync(releasesJsonPath); + var releasesFile = JsonSerializer.Deserialize(releasesJson, SerializerOptions)!; // TODO + + foreach (var release in releasesFile.Releases) + { + if (release.Sdk.Version is null) + { + warnings.Add($"Skipping release with missing version information from {releasesJson}"); + continue; + } + + if (release.Sdk.RuntimeVersion is null) + { + warnings.Add($"Skipping release with missing runtime version information from {releasesJson}"); + continue; + } + + if (!sdkPackages.Packages.TryGetValue(release.Sdk.Version, out var packagesAndVersions)) + { + packagesAndVersions = new PackageSet(); + sdkPackages.Packages[release.Sdk.Version] = packagesAndVersions; + } + + var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), release.Sdk.RuntimeVersion.ToString())); + var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{release.Sdk.RuntimeVersion}.md"); + if (!File.Exists(runtimeMarkdownPath)) + { + warnings.Add($"Unable to find expected markdown file {runtimeMarkdownPath}"); + continue; + } + + var markdownContent = await File.ReadAllTextAsync(runtimeMarkdownPath); + var packages = GetPackagesFromMarkdown(runtimeMarkdownPath, markdownContent, warnings); + foreach (var (packageName, packageVersion) in packages) + { + packagesAndVersions.Packages[packageName] = packageVersion; + } + } + } + + return (sdkPackages, warnings); + } + + public static ImmutableArray<(string Name, SemVersion Version)> GetPackagesFromMarkdown(string markdownPath, string markdownContent, List warnings) + { + var lines = markdownContent.Split("\n").Select(l => l.Trim()).ToArray(); + + // the markdown file contains a table that looks like this: + // Package name | Version + // :----------- | :------------------ + // Some.Package | 1.2.3 + // ... + // however there are some formatting issues with some elements that prevent markdown parsers from + // discovering it, so we fall back to manual parsing + + var tableStartLine = -1; + for (int i = 0; i < lines.Length; i++) + { + if (Regex.IsMatch(lines[i], "Package name.*Version")) + { + tableStartLine = i; + break; + } + } + + if (tableStartLine == -1) + { + warnings.Add($"Unable to find table start in file {markdownPath}"); + return []; + } + + // skip the column names and separator line + tableStartLine += 2; + + var tableEndLine = lines.Length; // assume the end of the file unless we find a blank line + for (int i = tableStartLine; i < lines.Length; i++) + { + if (string.IsNullOrEmpty(lines[i])) + { + tableEndLine = i; + break; + } + } + + var packages = new List<(string Name, SemVersion Version)>(); + for (int i = tableStartLine; i < tableEndLine; i++) + { + var line = lines[i].Trim(); + var foundMatch = false; + foreach (var pattern in SpecialCasePatterns) + { + var match = pattern.Match(line); + if (match.Success) + { + var packageName = match.Groups["PackageName"].Value; + var packageVersionString = match.Groups["PackageVersion"].Value; + if (SemVersion.TryParse(packageVersionString, out var packageVersion)) + { + packages.Add((packageName, packageVersion)); + foundMatch = true; + break; ; + } + } + } + + if (!foundMatch) + { + warnings.Add($"Unable to parse package and version from string [{line}] in file [{markdownPath}]:{i}"); + } + } + + return packages.ToImmutableArray(); + } + + // The different patterns the lines in the markdown might take. Due to issues with regular expressions, this list + // is in a very specific order. + private static ImmutableArray SpecialCasePatterns { get; } = [ + StandardLineWithFileExtensions(), + StandardLine(), + PackageNameDotVersion(), + PackageFileNameWithOptionalTrailingPipe(), + MultiColumnWithOptionalFileSuffix(), + ]; + + [GeneratedRegex(@"^(?[^|\s]+)\s*\|\s*(?[^|\s]+?)(\.symbols)?\.nupkg$", RegexOptions.Compiled)] + // Some.Package | 1.2.3.nupkg + // Some.Package | 1.2.3.symbols.nupkg + private static partial Regex StandardLineWithFileExtensions(); + + [GeneratedRegex(@"^(?[^|\s]+)\s*\|\s*(?[^|\s]+)$", RegexOptions.Compiled)] + // Some.Package | 1.2.3 + private static partial Regex StandardLine(); + + [GeneratedRegex(@"^(?[^\d]+)\.(?[\d].+)$", RegexOptions.Compiled)] + // Some.Package.1.2.3 + private static partial Regex PackageNameDotVersion(); + + [GeneratedRegex(@"^(?[^\d]+)\.(?\d.+?)\.nupkg(\s+\|)?$", RegexOptions.Compiled)] + // some.package.1.2.3.nupkg + // some.package.1.2.3.nupkg | + private static partial Regex PackageFileNameWithOptionalTrailingPipe(); + + [GeneratedRegex(@"^(?[^|\s]+)\s*\|[^|]*\|\s*(?.*?)(\.nupkg)?$", RegexOptions.Compiled)] + // Some.Package | 1.2 | 1.2.3.nupkg + private static partial Regex MultiColumnWithOptionalFileSuffix(); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj new file mode 100644 index 0000000000..3672a3eaee --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj @@ -0,0 +1,14 @@ + + + + $(CommonTargetFramework) + Exe + + + + + + + + + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs new file mode 100644 index 0000000000..9c017ced8e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs @@ -0,0 +1,8 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public record PackageSet +{ + public SortedDictionary Packages { get; init; } = new(StringComparer.OrdinalIgnoreCase); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs new file mode 100644 index 0000000000..13788e698f --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace DotNetPackageCorrelation; + +public record Release +{ + [JsonPropertyName("sdk")] + public required Sdk Sdk { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs new file mode 100644 index 0000000000..a1c6192182 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace DotNetPackageCorrelation; + +public record ReleasesFile +{ + [JsonPropertyName("releases")] + public required Release[] Releases { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs new file mode 100644 index 0000000000..500a3746db --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs @@ -0,0 +1,12 @@ +using Semver; +using System.Text.Json.Serialization; + +namespace DotNetPackageCorrelation; + +public record Sdk +{ + [JsonPropertyName("version")] + public required SemVersion? Version { get; init; } + [JsonPropertyName("runtime-version")] + public SemVersion? RuntimeVersion { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs new file mode 100644 index 0000000000..b931e4bbdf --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs @@ -0,0 +1,8 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public record SdkPackages +{ + public SortedDictionary Packages { get; init; } = new(new SemVerComparer()); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs new file mode 100644 index 0000000000..31a258922e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs @@ -0,0 +1,14 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public class SemVerComparer : IComparer +{ + public int Compare(SemVersion? x, SemVersion? y) + { + ArgumentNullException.ThrowIfNull(x); + ArgumentNullException.ThrowIfNull(y); + + return x.CompareSortOrderTo(y); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs new file mode 100644 index 0000000000..95829ebc98 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs @@ -0,0 +1,31 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + +using Semver; + +namespace DotNetPackageCorrelation; + +public class SemVersionConverter : JsonConverter +{ + public override SemVersion? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = reader.GetString(); + if (SemVersion.TryParse(value, out var result)) + { + return result; + } + + return null; + } + + public override void Write(Utf8JsonWriter writer, SemVersion? value, JsonSerializerOptions options) + { + writer.WriteStringValue(value?.ToString()); + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, [DisallowNull] SemVersion value, JsonSerializerOptions options) + { + writer.WritePropertyName(value.ToString()); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs new file mode 100644 index 0000000000..65e8ca248a --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs @@ -0,0 +1,30 @@ +using System.CommandLine; +using System.Text.Json; + +namespace DotNetPackageCorrelation; + +public class Program +{ + public static async Task Main(string[] args) + { + var coreLocationOption = new Option("--core-location", "The location of the .NET Core source code.") { IsRequired = true }; + var outputOption = new Option("--output", "The location to write the result.") { IsRequired = true }; + var command = new Command("build") + { + coreLocationOption, + outputOption, + }; + command.TreatUnmatchedTokensAsErrors = true; + command.SetHandler(async (coreLocationDirectory, output) => + { + // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory + var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes")); + var correlator = new Correlator(releaseNotesDirectory); + var result = await correlator.RunAsync(); + var json = JsonSerializer.Serialize(result, Correlator.SerializerOptions); + await File.WriteAllTextAsync(output.FullName, json); + }, coreLocationOption, outputOption); + var exitCode = await command.InvokeAsync(args); + return exitCode; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln index 819cbc2597..f4889927c0 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln @@ -1,4 +1,3 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.33516.290 @@ -43,6 +42,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NuGet.Versioning", "NuGetPr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NuGetUpdater.Cli.Test", "NuGetUpdater.Cli.Test\NuGetUpdater.Cli.Test.csproj", "{BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation", "DotNetPackageCorrelation\DotNetPackageCorrelation.csproj", "{52A6437B-7E72-4CCF-8E1E-355000F5DC10}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation.Test", "DotNetPackageCorrelation.Test\DotNetPackageCorrelation.Test.csproj", "{0945703C-C8DC-44F0-B1D8-0EFE011411AE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -125,6 +128,14 @@ Global {BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}.Debug|Any CPU.Build.0 = Debug|Any CPU {BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}.Release|Any CPU.ActiveCfg = Release|Any CPU {BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}.Release|Any CPU.Build.0 = Release|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Release|Any CPU.Build.0 = Release|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/nuget/script/ci-test b/nuget/script/ci-test index 2c6687ed1b..352dfe3952 100755 --- a/nuget/script/ci-test +++ b/nuget/script/ci-test @@ -12,6 +12,7 @@ pushd ./helpers/lib/NuGetUpdater dotnet restore dotnet format --no-restore --exclude ../NuGet.Client --verify-no-changes -v diag dotnet build --configuration Release +dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./NuGetUpdater.Cli.Test/NuGetUpdater.Cli.Test.csproj dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./NuGetUpdater.Core.Test/NuGetUpdater.Core.Test.csproj popd From 6f18f1387fd1a6ca6ac35bee6e30e28b2942d854 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 11:17:50 -0700 Subject: [PATCH 02/13] build package correlation and include in updater --- .gitmodules | 3 +++ nuget/helpers/lib/NuGetUpdater/.gitignore | 1 + .../DotNetPackageCorrelation/Program.cs | 4 ++-- .../EnsureDotNetPackageCorrelation.targets | 24 +++++++++++++++++++ .../NuGetUpdater.Core.csproj | 2 ++ nuget/helpers/lib/dotnet-core | 1 + nuget/script/ci-test | 2 +- 7 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets create mode 160000 nuget/helpers/lib/dotnet-core diff --git a/.gitmodules b/.gitmodules index d30b825732..3e20502e0f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = nuget/helpers/lib/NuGet.Client url = https://github.com/NuGet/NuGet.Client branch = release-6.12.x +[submodule "nuget/helpers/lib/dotnet-core"] + path = nuget/helpers/lib/dotnet-core + url = https://github.com/dotnet/core diff --git a/nuget/helpers/lib/NuGetUpdater/.gitignore b/nuget/helpers/lib/NuGetUpdater/.gitignore index 6b39588b66..3a80b6dc44 100644 --- a/nuget/helpers/lib/NuGetUpdater/.gitignore +++ b/nuget/helpers/lib/NuGetUpdater/.gitignore @@ -4,4 +4,5 @@ bin/ obj/ Properties/launchSettings.json NuGetUpdater.sln.DotSettings.user +NuGetUpdater.Core/dotnet-package-correlation.json *.binlog diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs index 65e8ca248a..db0c576f7c 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs @@ -20,8 +20,8 @@ public static async Task Main(string[] args) // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes")); var correlator = new Correlator(releaseNotesDirectory); - var result = await correlator.RunAsync(); - var json = JsonSerializer.Serialize(result, Correlator.SerializerOptions); + var (packages, _warnings) = await correlator.RunAsync(); + var json = JsonSerializer.Serialize(packages, Correlator.SerializerOptions); await File.WriteAllTextAsync(output.FullName, json); }, coreLocationOption, outputOption); var exitCode = await command.InvokeAsync(args); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets new file mode 100644 index 0000000000..42e20bebcb --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets @@ -0,0 +1,24 @@ + + + + $(MSBuildThisFileDirectory)..\DotNetPackageCorrelation + $(MSBuildThisFileDirectory)..\..\dotnet-core + $(MSBuildThisFileDirectory)dotnet-package-correlation.json + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj index d45ed65905..7f20b269f7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj @@ -30,4 +30,6 @@ + + diff --git a/nuget/helpers/lib/dotnet-core b/nuget/helpers/lib/dotnet-core new file mode 160000 index 0000000000..649080cc3a --- /dev/null +++ b/nuget/helpers/lib/dotnet-core @@ -0,0 +1 @@ +Subproject commit 649080cc3a0e927abb88d05591256e23c29e2170 diff --git a/nuget/script/ci-test b/nuget/script/ci-test index 352dfe3952..0b8e620aa3 100755 --- a/nuget/script/ci-test +++ b/nuget/script/ci-test @@ -10,7 +10,7 @@ popd # C# unit tests pushd ./helpers/lib/NuGetUpdater dotnet restore -dotnet format --no-restore --exclude ../NuGet.Client --verify-no-changes -v diag +dotnet format --no-restore --exclude ../NuGet.Client --exclude ../dotnet-core --verify-no-changes -v diag dotnet build --configuration Release dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./NuGetUpdater.Cli.Test/NuGetUpdater.Cli.Test.csproj From 7182240d1dd38f70f111c87a090d5957508480b2 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 13:37:20 -0700 Subject: [PATCH 03/13] incorporate sdk package correlation --- .../CorrelatorTests.cs | 2 +- .../EndToEndTests.cs | 31 +++ .../SdkPackagesTests.cs | 205 ++++++++++++++++++ .../DotNetPackageCorrelation/Correlator.cs | 59 ++--- .../Model/PackageSet.cs | 3 + .../DotNetPackageCorrelation/Model/Release.cs | 16 ++ .../Model/SdkPackages.cs | 5 +- .../Model/SemVerComparer.cs | 4 +- .../Model/SemVersionConverter.cs | 11 + .../SdkPackagesExtensions.cs | 28 +++ .../EntryPointTests.Discover.cs | 70 ++++++ .../Discover/SdkProjectDiscovery.cs | 44 +++- .../EnsureDotNetPackageCorrelation.targets | 3 +- .../NuGetUpdater.Core.csproj | 1 + 14 files changed, 447 insertions(+), 35 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index b24dfe7c4b..5400bbf2fe 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -7,7 +7,7 @@ namespace DotNetPackageCorrelation.Tests; public class CorrelatorTests { [Fact] - public async Task AllFilesShapedAppropriately() + public async Task FileHandling_AllFilesShapedAppropriately() { // the JSON and markdown are shaped as expected var (packages, warnings) = await RunFromFilesAsync( diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs new file mode 100644 index 0000000000..b04f882a81 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs @@ -0,0 +1,31 @@ +using System.Runtime.CompilerServices; + +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation.Tests; + +public class EndToEndTests +{ + [Fact] + public async Task IntegrationTest() + { + // arrange + var thisFileDirectory = Path.GetDirectoryName(GetThisFilePath())!; + var dotnetCoreDirectory = Path.Combine(thisFileDirectory, "..", "..", "dotnet-core"); + var correlator = new Correlator(new DirectoryInfo(Path.Combine(dotnetCoreDirectory, "release-notes"))); + + // act + var (packages, _warnings) = await correlator.RunAsync(); + var sdkVersion = SemVersion.Parse("8.0.307"); + + // SDK 8.0.307 has no System.Text.Json, but 8.0.306 provides System.Text.Json 8.0.5 + var systemTextJsonPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); + + // assert + Assert.Equal("8.0.5", systemTextJsonPackageVersion?.ToString()); + } + + private static string GetThisFilePath([CallerFilePath] string? path = null) => path ?? throw new ArgumentNullException(nameof(path)); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs new file mode 100644 index 0000000000..9e1a0b136e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs @@ -0,0 +1,205 @@ +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation; + +public class SdkPackagesTests +{ + [Theory] + [MemberData(nameof(CorrelatedPackageCanBeFoundData))] + public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionString, string packageName, string? expectedPackageVersionString) + { + var sdkVersion = SemVersion.Parse(sdkVersionString); + var actualReplacementPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, packageName); + var expectedPackageVersion = expectedPackageVersionString is not null + ? SemVersion.Parse(expectedPackageVersionString) + : null; + Assert.Equal(expectedPackageVersion, actualReplacementPackageVersion); + } + + public static IEnumerable CorrelatedPackageCanBeFoundData() + { + // package not found in current sdk, but is in parent; more recent sdk has package, but that's not returned + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + { + SemVersion.Parse("1.0.102"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.2") } + } + } + }, + } + }, + // sdkVersionString + "1.0.101", + // packageName + "Some.Package", + // expectedPackageVersionString + "1.0.1" + ]; + + // package differing in case is found + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "some.package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + } + }, + // sdkVersionString + "1.0.101", + // packageName + "Some.Package", + // expectedPackageVersionString + "1.0.1" + ]; + + // package not found results in null version + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + } + }, + // sdkVersionString + "1.0.101", + // packageName + "UnrelatedPackage", + // expectedPackageVersionString + null + ]; + + // only SDKs with matching major version are considered + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.0") } + } + } + }, + { + SemVersion.Parse("2.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("2.0.1") } + } + } + }, + { + SemVersion.Parse("2.0.200"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + { + SemVersion.Parse("3.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("3.0.1") } + } + } + }, + } + }, + // sdkVersionString + "2.0.200", + // packageName + "Some.Package", + // expectedPackageVersionString + "2.0.1" + ]; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs index 01c0d131b6..cf6e89bcd5 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -8,7 +8,7 @@ namespace DotNetPackageCorrelation; public partial class Correlator { - internal static readonly JsonSerializerOptions SerializerOptions = new() + public static readonly JsonSerializerOptions SerializerOptions = new() { WriteIndented = true, Converters = { new SemVersionConverter() }, @@ -45,41 +45,44 @@ public Correlator(DirectoryInfo releaseNotesDirectory) } var releasesJson = await File.ReadAllTextAsync(releasesJsonPath); - var releasesFile = JsonSerializer.Deserialize(releasesJson, SerializerOptions)!; // TODO + var releasesFile = JsonSerializer.Deserialize(releasesJson, SerializerOptions)!; foreach (var release in releasesFile.Releases) { - if (release.Sdk.Version is null) + foreach (var sdk in release.GetSdks()) { - warnings.Add($"Skipping release with missing version information from {releasesJson}"); - continue; - } + if (sdk.Version is null) + { + warnings.Add($"Skipping release with missing version information from {releasesJson}"); + continue; + } - if (release.Sdk.RuntimeVersion is null) - { - warnings.Add($"Skipping release with missing runtime version information from {releasesJson}"); - continue; - } + if (sdk.RuntimeVersion is null) + { + warnings.Add($"Skipping release with missing runtime version information from {releasesJson}"); + continue; + } - if (!sdkPackages.Packages.TryGetValue(release.Sdk.Version, out var packagesAndVersions)) - { - packagesAndVersions = new PackageSet(); - sdkPackages.Packages[release.Sdk.Version] = packagesAndVersions; - } + if (!sdkPackages.Packages.TryGetValue(sdk.Version, out var packagesAndVersions)) + { + packagesAndVersions = new PackageSet(); + sdkPackages.Packages[sdk.Version] = packagesAndVersions; + } - var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), release.Sdk.RuntimeVersion.ToString())); - var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{release.Sdk.RuntimeVersion}.md"); - if (!File.Exists(runtimeMarkdownPath)) - { - warnings.Add($"Unable to find expected markdown file {runtimeMarkdownPath}"); - continue; - } + var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), sdk.RuntimeVersion.ToString())); + var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{sdk.RuntimeVersion}.md"); + if (!File.Exists(runtimeMarkdownPath)) + { + warnings.Add($"Unable to find expected markdown file {runtimeMarkdownPath}"); + continue; + } - var markdownContent = await File.ReadAllTextAsync(runtimeMarkdownPath); - var packages = GetPackagesFromMarkdown(runtimeMarkdownPath, markdownContent, warnings); - foreach (var (packageName, packageVersion) in packages) - { - packagesAndVersions.Packages[packageName] = packageVersion; + var markdownContent = await File.ReadAllTextAsync(runtimeMarkdownPath); + var packages = GetPackagesFromMarkdown(runtimeMarkdownPath, markdownContent, warnings); + foreach (var (packageName, packageVersion) in packages) + { + packagesAndVersions.Packages[packageName] = packageVersion; + } } } } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs index 9c017ced8e..90461f6b39 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs @@ -1,8 +1,11 @@ +using System.Text.Json.Serialization; + using Semver; namespace DotNetPackageCorrelation; public record PackageSet { + [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] public SortedDictionary Packages { get; init; } = new(StringComparer.OrdinalIgnoreCase); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs index 13788e698f..010c3dc4b7 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using System.Text.Json.Serialization; namespace DotNetPackageCorrelation; @@ -6,4 +7,19 @@ public record Release { [JsonPropertyName("sdk")] public required Sdk Sdk { get; init; } + + [JsonPropertyName("sdks")] + public ImmutableArray? Sdks { get; init; } = []; +} + +public static class ReleaseExtensions +{ + public static IEnumerable GetSdks(this Release release) + { + yield return release.Sdk; + foreach (var sdk in release.Sdks ?? []) + { + yield return sdk; + } + } } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs index b931e4bbdf..ba51a349f9 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs @@ -1,8 +1,11 @@ +using System.Text.Json.Serialization; + using Semver; namespace DotNetPackageCorrelation; public record SdkPackages { - public SortedDictionary Packages { get; init; } = new(new SemVerComparer()); + [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] + public SortedDictionary Packages { get; init; } = new(SemVerComparer.Instance); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs index 31a258922e..bd130d9628 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs @@ -4,11 +4,13 @@ namespace DotNetPackageCorrelation; public class SemVerComparer : IComparer { + public static SemVerComparer Instance = new(); + public int Compare(SemVersion? x, SemVersion? y) { ArgumentNullException.ThrowIfNull(x); ArgumentNullException.ThrowIfNull(y); - return x.CompareSortOrderTo(y); + return x.ComparePrecedenceTo(y); } } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs index 95829ebc98..43b43c7dc5 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs @@ -19,6 +19,17 @@ public class SemVersionConverter : JsonConverter return null; } + public override SemVersion ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = Read(ref reader, typeToConvert, options); + if (value is null) + { + throw new JsonException($"Unable to read {nameof(SemVersion)} as property name."); + } + + return value; + } + public override void Write(Utf8JsonWriter writer, SemVersion? value, JsonSerializerOptions options) { writer.WriteStringValue(value?.ToString()); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs new file mode 100644 index 0000000000..308be1dd35 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs @@ -0,0 +1,28 @@ +using System.Collections.Immutable; + +using Semver; + +namespace DotNetPackageCorrelation; + +public static class SdkPackagesExtensions +{ + public static SemVersion? GetReplacementPackageVersion(this SdkPackages packages, SemVersion sdkVersion, string packageName) + { + var sdkVersionsToCheck = packages.Packages.Keys + .Where(v => v.Major == sdkVersion.Major) + .Where(v => v.ComparePrecedenceTo(sdkVersion) <= 0) + .OrderBy(v => v, SemVerComparer.Instance) + .Reverse() + .ToImmutableArray(); + foreach (var sdkVersionToCheck in sdkVersionsToCheck) + { + var sdkPackages = packages.Packages[sdkVersionToCheck]; + if (sdkPackages.Packages.TryGetValue(packageName, out var packageVersion)) + { + return packageVersion; + } + } + + return null; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index ad43a2292d..e31ea8136b 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -388,6 +388,76 @@ await RunAsync(path => ); } + [Fact] + public async Task SdkManagedPackagesAreAppropriatelyReturned() + { + // this test uses live packages + + // .NET SDK 8.0.306 ships with System.Text.Json/8.0.5 + await RunAsync(path => + [ + "discover", + "--job-path", + Path.Combine(path, "job.json"), + "--repo-root", + path, + "--workspace", + "src", + "--output", + Path.Combine(path, DiscoveryWorker.DiscoveryResultFileName) + ], + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + initialFiles: [ + ("src/global.json", """ + { + "sdk": { + "version": "8.0.307", + "rollForward": "latestMinor" + } + } + """), + ("src/project.csproj", """ + + + net8.0 + + + + + + """) + ], + expectedResult: new() + { + Path = "src", + Projects = [ + new() + { + FilePath = "project.csproj", + Dependencies = [ + new("System.Text.Encodings.Web", "8.0.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true), + new("System.Text.Json", "8.0.5", DependencyType.PackageReference, TargetFrameworks: ["net8.0"], IsDirect: true), + ], + Properties = [ + new("TargetFramework", "net8.0", "src/project.csproj"), + ], + TargetFrameworks = ["net8.0"], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [], + } + ], + GlobalJson = new() + { + FilePath = "global.json", + Dependencies = [ + new("Microsoft.NET.Sdk", "8.0.307", DependencyType.MSBuildSdk), + ] + } + } + ); + } + private static async Task RunAsync( Func getArgs, TestFile[] initialFiles, diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 266d746fa3..b20c085df9 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -1,20 +1,34 @@ using System.Collections.Immutable; using System.Reflection; +using System.Text.Json; using System.Xml.Linq; using System.Xml.XPath; +using DotNetPackageCorrelation; + using Microsoft.Build.Logging.StructuredLogger; using NuGet.Versioning; using NuGetUpdater.Core.Utilities; +using Semver; + using LoggerProperty = Microsoft.Build.Logging.StructuredLogger.Property; namespace NuGetUpdater.Core.Discover; internal static class SdkProjectDiscovery { + private static readonly SdkPackages _sdkPackages; + + static SdkProjectDiscovery() + { + var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); + var packageCorrelationJson = File.ReadAllText(packageCorrelationPath); + _sdkPackages = JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + } + private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) { "PackageReference" @@ -164,7 +178,7 @@ public static async Task> DiscoverWithBin } break; case NamedNode namedNode when namedNode is AddItem or RemoveItem: - ProcessResolvedPackageReference(namedNode, packagesPerProject, topLevelPackagesPerProject); + ProcessResolvedPackageReference(namedNode, packagesPerProject, topLevelPackagesPerProject, experimentsManager); if (namedNode is AddItem addItem) { @@ -285,10 +299,18 @@ public static async Task> DiscoverWithBin return projectDiscoveryResults; } + private static string? GetCorrespondingSdkManagedPackageVersion(string packageName, string sdkVersionString) + { + var sdkVersion = SemVersion.Parse(sdkVersionString); + var replacementPackageVersion = _sdkPackages.GetReplacementPackageVersion(sdkVersion, packageName); + return replacementPackageVersion?.ToString(); + } + private static void ProcessResolvedPackageReference( NamedNode node, Dictionary>> packagesPerProject, // projectPath -> tfm -> (packageName, packageVersion) - Dictionary> topLevelPackagesPerProject + Dictionary> topLevelPackagesPerProject, + ExperimentsManager experimentsManager ) { var doRemoveOperation = node is RemoveItem; @@ -348,7 +370,23 @@ Dictionary> topLevelPackagesPerProject if (doRemoveOperation) { - packagesPerTfm.Remove(packageName); + var wasRemoved = packagesPerTfm.Remove(packageName); + if (wasRemoved) + { + if (experimentsManager.InstallDotnetSdks) + { + // dotnet package correlation correction requires specific dotnet sdk handling + var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); + if (sdkVersionString is not null) + { + var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); + if (replacementVersion is not null) + { + packagesPerTfm[packageName] = replacementVersion.ToString(); + } + } + } + } } if (doAddOperation) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets index 42e20bebcb..031c1fd050 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets @@ -8,9 +8,10 @@ + <_DotNetCoreFiles Include="$(DotNetCoreDirectory)\**\*" /> - + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj index 7f20b269f7..297314383b 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj @@ -14,6 +14,7 @@ + From 2c4d06d2604bc347e99a7db3b5fa851def5d9b0a Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 14:01:31 -0700 Subject: [PATCH 04/13] add update tests --- .../UpdateWorkerTests.PackageReference.cs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index c862d6cc93..e8dcfc56c1 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3487,5 +3487,84 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", } ); } + + [Fact] + public async Task UpdateSdkManagedPackage_DirectDependency() + { + // this test uses live packages + await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.5", + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + projectContents: """ + + + net8.0 + + + + + + """, + additionalFiles: [ + ("global.json", """ + { + "sdk": { + "version": "8.0.307", + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: """ + + + net8.0 + + + + + + """ + ); + } + + [Fact] + public async Task UpdateSdkManagedPackage_TransitiveDependency() + { + // this test uses live packages + await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: true, + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + projectContents: """ + + + net8.0 + + + + + + """, + additionalFiles: [ + ("global.json", """ + { + "sdk": { + "version": "8.0.307", + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: """ + + + net8.0 + + + + + + + """ + ); + } } } From 47a8f210b6f293ae7244e576e4634aedac173243 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 14:06:10 -0700 Subject: [PATCH 05/13] add comment --- .../NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index e31ea8136b..3e6920c3c0 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -393,6 +393,7 @@ public async Task SdkManagedPackagesAreAppropriatelyReturned() { // this test uses live packages + // .NET SDK 8.0.303 ships with System.Text.Json/8.0.4 // .NET SDK 8.0.306 ships with System.Text.Json/8.0.5 await RunAsync(path => [ From 7118eca28e5be84030c1e4ae0e33ad69fa6d786b Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 14:24:34 -0700 Subject: [PATCH 06/13] fix typo --- .../DotNetPackageCorrelation.Test/CorrelatorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index 5400bbf2fe..82beda5a15 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -60,7 +60,7 @@ Package name | Version private static void AssertPackageVersion(SdkPackages packages, string sdkVersion, string packageName, string expectedPackageVersion) { - Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK verison [{sdkVersion}]"); + Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); var actualPackageVersion = packageVersion.ToString(); Assert.Equal(expectedPackageVersion, actualPackageVersion); From 80c23eb1e125ebface83b5b3e5de604df4faf8de Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 10:42:07 -0700 Subject: [PATCH 07/13] create separate cli for package correlation --- .../DotNetPackageCorrelation.Cli.csproj | 16 ++++++++++++++++ .../Program.cs | 4 +++- .../DotNetPackageCorrelation.csproj | 2 -- .../EnsureDotNetPackageCorrelation.targets | 4 ++-- nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln | 6 ++++++ 5 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj rename nuget/helpers/lib/NuGetUpdater/{DotNetPackageCorrelation => DotNetPackageCorrelation.Cli}/Program.cs (94%) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj new file mode 100644 index 0000000000..8e99811c84 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj @@ -0,0 +1,16 @@ + + + + $(CommonTargetFramework) + Exe + + + + + + + + + + + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs similarity index 94% rename from nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs rename to nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs index db0c576f7c..3caca06364 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs @@ -1,7 +1,9 @@ using System.CommandLine; using System.Text.Json; -namespace DotNetPackageCorrelation; +using DotNetPackageCorrelation; + +namespace DotNetPackageCorrelation.Cli; public class Program { diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj index 3672a3eaee..8d631d5aeb 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj @@ -2,12 +2,10 @@ $(CommonTargetFramework) - Exe - diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets index 031c1fd050..b24ce2967f 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets @@ -1,7 +1,7 @@ - $(MSBuildThisFileDirectory)..\DotNetPackageCorrelation + $(MSBuildThisFileDirectory)..\DotNetPackageCorrelation.Cli $(MSBuildThisFileDirectory)..\..\dotnet-core $(MSBuildThisFileDirectory)dotnet-package-correlation.json @@ -12,7 +12,7 @@ - + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln index f4889927c0..6fa3f0a93d 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln @@ -46,6 +46,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation.Test", "DotNetPackageCorrelation.Test\DotNetPackageCorrelation.Test.csproj", "{0945703C-C8DC-44F0-B1D8-0EFE011411AE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation.Cli", "DotNetPackageCorrelation.Cli\DotNetPackageCorrelation.Cli.csproj", "{509454EE-629F-4767-B1D4-7F2DF86C11B5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -136,6 +138,10 @@ Global {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Debug|Any CPU.Build.0 = Debug|Any CPU {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.ActiveCfg = Release|Any CPU {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.Build.0 = Release|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From c74f3ee8673ec24df5111038a786e0760a8c48fb Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 10:58:29 -0700 Subject: [PATCH 08/13] fix publishing --- .../DotNetPackageCorrelation/Model/Sdk.cs | 3 ++- .../Discover/SdkProjectDiscovery.cs | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs index 500a3746db..a2098ec883 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs @@ -1,6 +1,7 @@ -using Semver; using System.Text.Json.Serialization; +using Semver; + namespace DotNetPackageCorrelation; public record Sdk diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index b20c085df9..4d7718162b 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -373,16 +373,22 @@ ExperimentsManager experimentsManager var wasRemoved = packagesPerTfm.Remove(packageName); if (wasRemoved) { - if (experimentsManager.InstallDotnetSdks) + // we only want to track packages that were explicitly part of the SDK's conflict resolution + if (node.Name == "RuntimeCopyLocalItems" && + node.Parent is Target target && + target.Name == "GenerateBuildDependencyFile") { // dotnet package correlation correction requires specific dotnet sdk handling - var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); - if (sdkVersionString is not null) + if (experimentsManager.InstallDotnetSdks) { - var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); - if (replacementVersion is not null) + var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); + if (sdkVersionString is not null) { - packagesPerTfm[packageName] = replacementVersion.ToString(); + var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); + if (replacementVersion is not null) + { + packagesPerTfm[packageName] = replacementVersion.ToString(); + } } } } From 90607ec926ba4c9863fa63b2174d5c8c7bf41cc5 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 13:20:52 -0700 Subject: [PATCH 09/13] make unit test SDK agnostic --- .../EntryPointTests.Discover.cs | 71 --------------- .../Discover/DiscoveryWorkerTests.Project.cs | 88 +++++++++++++++++++ .../Discover/SdkProjectDiscovery.cs | 31 ++++++- 3 files changed, 116 insertions(+), 74 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index 3e6920c3c0..ad43a2292d 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -388,77 +388,6 @@ await RunAsync(path => ); } - [Fact] - public async Task SdkManagedPackagesAreAppropriatelyReturned() - { - // this test uses live packages - - // .NET SDK 8.0.303 ships with System.Text.Json/8.0.4 - // .NET SDK 8.0.306 ships with System.Text.Json/8.0.5 - await RunAsync(path => - [ - "discover", - "--job-path", - Path.Combine(path, "job.json"), - "--repo-root", - path, - "--workspace", - "src", - "--output", - Path.Combine(path, DiscoveryWorker.DiscoveryResultFileName) - ], - experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, - initialFiles: [ - ("src/global.json", """ - { - "sdk": { - "version": "8.0.307", - "rollForward": "latestMinor" - } - } - """), - ("src/project.csproj", """ - - - net8.0 - - - - - - """) - ], - expectedResult: new() - { - Path = "src", - Projects = [ - new() - { - FilePath = "project.csproj", - Dependencies = [ - new("System.Text.Encodings.Web", "8.0.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true), - new("System.Text.Json", "8.0.5", DependencyType.PackageReference, TargetFrameworks: ["net8.0"], IsDirect: true), - ], - Properties = [ - new("TargetFramework", "net8.0", "src/project.csproj"), - ], - TargetFrameworks = ["net8.0"], - ReferencedProjectPaths = [], - ImportedFiles = [], - AdditionalFiles = [], - } - ], - GlobalJson = new() - { - FilePath = "global.json", - Dependencies = [ - new("Microsoft.NET.Sdk", "8.0.307", DependencyType.MSBuildSdk), - ] - } - } - ); - } - private static async Task RunAsync( Func getArgs, TestFile[] initialFiles, diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index fc0defa228..9d3cb0b0d1 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1263,5 +1263,93 @@ await TestDiscoveryAsync( } ); } + + [Fact] + public async Task PackagesManagedAndRemovedByTheSdkAreReported() + { + // To avoid a unit test that's tightly coupled to the installed SDK, some files are faked. + // First up, the `dotnet-package-correlation.json` is faked to have the appropriate shape to report a + // package replacement. Doing this requires a temporary file and environment variable override. + using var tempDirectory = new TemporaryDirectory(); + var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); + await File.WriteAllTextAsync(packageCorrelationFile, """ + { + "Packages": { + "8.0.100": { + "Packages": { + "Test.Only.Package": "1.0.99" + } + } + } + } + """); + using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); + + // The SDK package handling is detected in a very specific circumstance; an assembly being removed from the + // `@(RuntimeCopyLocalItems)` item group in the `GenerateBuildDependencyFile` target. Since we don't want + // to involve the real SDK, we fake some required targets. + await TestDiscoveryAsync( + experimentsManager: new ExperimentsManager() { InstallDotnetSdks = true, UseDirectDiscovery = true }, + packages: [], + workspacePath: "", + files: + [ + ("project.csproj", """ + + + + + + + + + 8.0.100 + net8.0 + + + + + + + + + + + + + + + + + + + + + + """) + ], + expectedResult: new() + { + Path = "", + Projects = [ + new() + { + FilePath = "project.csproj", + Dependencies = [ + new("Test.Only.Package", "1.0.99", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true) + ], + Properties = [ + new("NETCoreSdkVersion", "8.0.100", "project.csproj"), + new("TargetFramework", "net8.0", "project.csproj"), + ], + TargetFrameworks = ["net8.0"], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [], + } + ] + } + ); + } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 4d7718162b..7131743238 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -21,12 +21,12 @@ namespace NuGetUpdater.Core.Discover; internal static class SdkProjectDiscovery { private static readonly SdkPackages _sdkPackages; + private static readonly Dictionary _sdkPackagesByOverrideFile = new(); static SdkProjectDiscovery() { var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); - var packageCorrelationJson = File.ReadAllText(packageCorrelationPath); - _sdkPackages = JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + _sdkPackages = LoadPackageCorrelationsFromFile(packageCorrelationPath); } private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) @@ -302,10 +302,35 @@ public static async Task> DiscoverWithBin private static string? GetCorrespondingSdkManagedPackageVersion(string packageName, string sdkVersionString) { var sdkVersion = SemVersion.Parse(sdkVersionString); - var replacementPackageVersion = _sdkPackages.GetReplacementPackageVersion(sdkVersion, packageName); + var replacementPackageVersion = GetSdkPackageCorrelations().GetReplacementPackageVersion(sdkVersion, packageName); return replacementPackageVersion?.ToString(); } + private static SdkPackages GetSdkPackageCorrelations() + { + var packageCorrelationFileOverride = Environment.GetEnvironmentVariable("DOTNET_PACKAGE_CORRELATION_FILE_PATH"); + if (packageCorrelationFileOverride is not null) + { + // this is used as a test hook to allow unit tests to be SDK agnostic + if (_sdkPackagesByOverrideFile.TryGetValue(packageCorrelationFileOverride, out var sdkPackages)) + { + return sdkPackages; + } + + sdkPackages = LoadPackageCorrelationsFromFile(packageCorrelationFileOverride); + _sdkPackagesByOverrideFile[packageCorrelationFileOverride] = sdkPackages; + return sdkPackages; + } + + return _sdkPackages; + } + + private static SdkPackages LoadPackageCorrelationsFromFile(string filePath) + { + var packageCorrelationJson = File.ReadAllText(filePath); + return JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + } + private static void ProcessResolvedPackageReference( NamedNode node, Dictionary>> packagesPerProject, // projectPath -> tfm -> (packageName, packageVersion) From 5599bca8777603e86d1ef87943300ab04b44310c Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 13:47:56 -0700 Subject: [PATCH 10/13] rename property --- .../DotNetPackageCorrelation.Cli/Program.cs | 4 ++-- .../DotNetPackageCorrelation.Test/CorrelatorTests.cs | 12 ++++++------ .../DotNetPackageCorrelation.Test/EndToEndTests.cs | 4 ++-- .../SdkPackagesTests.cs | 8 ++++---- .../DotNetPackageCorrelation/Correlator.cs | 6 +++--- .../DotNetPackageCorrelation/Model/SdkPackages.cs | 2 +- .../SdkPackagesExtensions.cs | 4 ++-- .../Discover/DiscoveryWorkerTests.Project.cs | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs index 3caca06364..b9e608c0ac 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs @@ -22,8 +22,8 @@ public static async Task Main(string[] args) // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes")); var correlator = new Correlator(releaseNotesDirectory); - var (packages, _warnings) = await correlator.RunAsync(); - var json = JsonSerializer.Serialize(packages, Correlator.SerializerOptions); + var (sdkPackages, _warnings) = await correlator.RunAsync(); + var json = JsonSerializer.Serialize(sdkPackages, Correlator.SerializerOptions); await File.WriteAllTextAsync(output.FullName, json); }, coreLocationOption, outputOption); var exitCode = await command.InvokeAsync(args); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index 82beda5a15..5efa04977f 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -10,7 +10,7 @@ public class CorrelatorTests public async Task FileHandling_AllFilesShapedAppropriately() { // the JSON and markdown are shaped as expected - var (packages, warnings) = await RunFromFilesAsync( + var (sdkPackages, warnings) = await RunFromFilesAsync( ("8.0/releases.json", """ { "releases": [ @@ -31,8 +31,8 @@ Package name | Version """) ); Assert.Empty(warnings); - AssertPackageVersion(packages, "8.0.100", "Package.A", "8.0.0"); - AssertPackageVersion(packages, "8.0.100", "Package.B", "1.2.3"); + AssertPackageVersion(sdkPackages, "8.0.100", "Package.A", "8.0.0"); + AssertPackageVersion(sdkPackages, "8.0.100", "Package.B", "1.2.3"); } [Theory] @@ -58,15 +58,15 @@ Package name | Version Assert.Equal(expectedPackageVersion, actualpackage.Version.ToString()); } - private static void AssertPackageVersion(SdkPackages packages, string sdkVersion, string packageName, string expectedPackageVersion) + private static void AssertPackageVersion(SdkPackages sdkPackages, string sdkVersion, string packageName, string expectedPackageVersion) { - Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); + Assert.True(sdkPackages.Sdks.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); var actualPackageVersion = packageVersion.ToString(); Assert.Equal(expectedPackageVersion, actualPackageVersion); } - private static async Task<(SdkPackages Packages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) + private static async Task<(SdkPackages SdkPackages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) { var testDirectory = Path.Combine(Path.GetDirectoryName(typeof(CorrelatorTests).Assembly.Location)!, "test-data", Guid.NewGuid().ToString("D")); Directory.CreateDirectory(testDirectory); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs index b04f882a81..3da1ceab2c 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs @@ -17,11 +17,11 @@ public async Task IntegrationTest() var correlator = new Correlator(new DirectoryInfo(Path.Combine(dotnetCoreDirectory, "release-notes"))); // act - var (packages, _warnings) = await correlator.RunAsync(); + var (sdkPackages, _warnings) = await correlator.RunAsync(); var sdkVersion = SemVersion.Parse("8.0.307"); // SDK 8.0.307 has no System.Text.Json, but 8.0.306 provides System.Text.Json 8.0.5 - var systemTextJsonPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); + var systemTextJsonPackageVersion = sdkPackages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); // assert Assert.Equal("8.0.5", systemTextJsonPackageVersion?.ToString()); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs index 9e1a0b136e..d6428e6683 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs @@ -26,7 +26,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), @@ -74,7 +74,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), @@ -112,7 +112,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), @@ -150,7 +150,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs index cf6e89bcd5..bf6df66445 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -21,7 +21,7 @@ public Correlator(DirectoryInfo releaseNotesDirectory) _releaseNotesDirectory = releaseNotesDirectory; } - public async Task<(SdkPackages Packages, IEnumerable Warnings)> RunAsync() + public async Task<(SdkPackages SdkPackages, IEnumerable Warnings)> RunAsync() { var runtimeVersions = new List(); foreach (var directory in Directory.EnumerateDirectories(_releaseNotesDirectory.FullName)) @@ -63,10 +63,10 @@ public Correlator(DirectoryInfo releaseNotesDirectory) continue; } - if (!sdkPackages.Packages.TryGetValue(sdk.Version, out var packagesAndVersions)) + if (!sdkPackages.Sdks.TryGetValue(sdk.Version, out var packagesAndVersions)) { packagesAndVersions = new PackageSet(); - sdkPackages.Packages[sdk.Version] = packagesAndVersions; + sdkPackages.Sdks[sdk.Version] = packagesAndVersions; } var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), sdk.RuntimeVersion.ToString())); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs index ba51a349f9..8ebfa6dfa7 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs @@ -7,5 +7,5 @@ namespace DotNetPackageCorrelation; public record SdkPackages { [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] - public SortedDictionary Packages { get; init; } = new(SemVerComparer.Instance); + public SortedDictionary Sdks { get; init; } = new(SemVerComparer.Instance); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs index 308be1dd35..a8d20db66a 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs @@ -8,7 +8,7 @@ public static class SdkPackagesExtensions { public static SemVersion? GetReplacementPackageVersion(this SdkPackages packages, SemVersion sdkVersion, string packageName) { - var sdkVersionsToCheck = packages.Packages.Keys + var sdkVersionsToCheck = packages.Sdks.Keys .Where(v => v.Major == sdkVersion.Major) .Where(v => v.ComparePrecedenceTo(sdkVersion) <= 0) .OrderBy(v => v, SemVerComparer.Instance) @@ -16,7 +16,7 @@ public static class SdkPackagesExtensions .ToImmutableArray(); foreach (var sdkVersionToCheck in sdkVersionsToCheck) { - var sdkPackages = packages.Packages[sdkVersionToCheck]; + var sdkPackages = packages.Sdks[sdkVersionToCheck]; if (sdkPackages.Packages.TryGetValue(packageName, out var packageVersion)) { return packageVersion; diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index 9d3cb0b0d1..4a8ccff93e 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1274,7 +1274,7 @@ public async Task PackagesManagedAndRemovedByTheSdkAreReported() var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); await File.WriteAllTextAsync(packageCorrelationFile, """ { - "Packages": { + "Sdks": { "8.0.100": { "Packages": { "Test.Only.Package": "1.0.99" From d22e986cbc8290b54b26076aca12d66d7969a1b5 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 14:37:49 -0700 Subject: [PATCH 11/13] fix typo --- .../Discover/DiscoveryWorkerTests.Project.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index 4a8ccff93e..04b9e6f274 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1309,7 +1309,7 @@ await TestDiscoveryAsync( - + From e5762ae74ff6f8b186d3e988c8d005c7c6564670 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 15:54:28 -0700 Subject: [PATCH 12/13] use local packages for test --- .../UpdateWorkerTests.PackageReference.cs | 72 ++++++++----------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index e8dcfc56c1..98dbd87eab 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3491,55 +3491,42 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", [Fact] public async Task UpdateSdkManagedPackage_DirectDependency() { - // this test uses live packages - await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.5", - experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, - projectContents: """ - - - net8.0 - - - - - - """, - additionalFiles: [ - ("global.json", """ - { - "sdk": { - "version": "8.0.307", - "rollForward": "latestMinor" + // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. + // Doing this requires a temporary file and environment variable override. Note that SDK version 8.0.100 + // or greater is required. + using var tempDirectory = new TemporaryDirectory(); + var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); + await File.WriteAllTextAsync(packageCorrelationFile, """ + { + "Sdks": { + "8.0.100": { + "Packages": { + "System.Text.Json": "8.0.98" } } - """) - ], - expectedProjectContents: """ - - - net8.0 - - - - - - """ - ); - } - - [Fact] - public async Task UpdateSdkManagedPackage_TransitiveDependency() - { - // this test uses live packages - await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: true, + } + } + """); + using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); + + // In the `packages` section below, we fake a `System.Text.Json` package with a low assembly version that + // will always trigger the replacement so that can be detected and then the equivalent version is pulled + // from the correlation file specified above. In the original project contents, package version `8.0.98` + // is reported which makes the update to `8.0.99` always possible. + await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.99", experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + packages: + [ + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.0", "net8.0", assemblyVersion: "8.0.0.0"), // this assembly version is lower than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.99", "net8.0", assemblyVersion: "8.99.99.99"), // this assembly version is greater than what the SDK will have + ], projectContents: """ net8.0 - + """, @@ -3547,7 +3534,7 @@ await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: ("global.json", """ { "sdk": { - "version": "8.0.307", + "version": "8.0.100", "rollForward": "latestMinor" } } @@ -3559,8 +3546,7 @@ await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: net8.0 - - + """ From 32de6aff4fea16c3f322cb4f56719198d8470a72 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 19:21:11 -0700 Subject: [PATCH 13/13] maintain `global.json` during dependency resolution --- .../UpdateWorkerTests.PackageReference.cs | 68 ++++++++++++++++ .../Utilities/MSBuildHelper.cs | 78 ++++++++++++------- .../Utilities/NuGetHelper.cs | 2 +- 3 files changed, 120 insertions(+), 28 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index 98dbd87eab..8c689dc4e3 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3552,5 +3552,73 @@ await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.99", """ ); } + + [Fact(Skip = "https://github.com/dependabot/dependabot-core/issues/11140")] + public async Task UpdateSdkManagedPackage_TransitiveDependency() + { + // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. + // Doing this requires a temporary file and environment variable override. Note that SDK version 8.0.100 + // or greater is required. + using var tempDirectory = new TemporaryDirectory(); + var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); + await File.WriteAllTextAsync(packageCorrelationFile, """ + { + "Sdks": { + "8.0.100": { + "Packages": { + "System.Text.Json": "8.0.98" + } + } + } + } + """); + using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); + + // In the `packages` section below, we fake a `System.Text.Json` package with a low assembly version that + // will always trigger the replacement so that can be detected and then the equivalent version is pulled + // from the correlation file specified above. In the original project contents, package version `8.0.98` + // is reported which makes the update to `8.0.99` always possible. + await TestUpdateForProject("System.Text.Json", "8.0.98", "8.0.99", + isTransitive: true, + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0", [(null, [("System.Text.Json", "[8.0.0]")])]), + MockNuGetPackage.CreateSimplePackage("Some.Package", "2.0.0", "net8.0", [(null, [("System.Text.Json", "[8.0.99]")])]), + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.0", "net8.0", assemblyVersion: "8.0.0.0"), // this assembly version is lower than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.99", "net8.0", assemblyVersion: "8.99.99.99"), // this assembly version is greater than what the SDK will have + ], + projectContents: """ + + + net8.0 + + + + + + """, + additionalFiles: [ + ("global.json", """ + { + "sdk": { + "version": "8.0.100", + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: """ + + + net8.0 + + + + + + """ + ); + } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs index 32dc812dbf..09a0b3d906 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs @@ -19,6 +19,7 @@ using NuGet.Versioning; using NuGetUpdater.Core.Analyze; +using NuGetUpdater.Core.Discover; using NuGetUpdater.Core.Utilities; namespace NuGetUpdater.Core; @@ -342,7 +343,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); try { - var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); // NU1608: Detected package version outside of dependency constraint @@ -362,7 +363,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s try { - string tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + string tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); // Add Dependency[] packages to List existingPackages @@ -518,7 +519,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); try { - var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); ThrowOnUnauthenticatedFeed(stdOut); @@ -668,12 +669,22 @@ internal static async Task CreateTempProjectAsync( string projectPath, string targetFramework, IReadOnlyCollection packages, + ExperimentsManager experimentsManager, ILogger logger, bool usePackageDownload = false) { var projectDirectory = Path.GetDirectoryName(projectPath); projectDirectory ??= repoRoot; - var topLevelFiles = Directory.GetFiles(repoRoot); + + if (experimentsManager.InstallDotnetSdks) + { + var globalJsonPath = PathHelper.GetFileInDirectoryOrParent(projectPath, repoRoot, "global.json", caseSensitive: true); + if (globalJsonPath is not null) + { + File.Copy(globalJsonPath, Path.Combine(tempDir.FullName, "global.json")); + } + } + var nugetConfigPath = PathHelper.GetFileInDirectoryOrParent(projectPath, repoRoot, "NuGet.Config", caseSensitive: false); if (nugetConfigPath is not null) { @@ -832,40 +843,53 @@ internal static async Task GetAllPackageDependenciesAsync( string targetFramework, IReadOnlyCollection packages, ExperimentsManager experimentsManager, - ILogger logger) + ILogger logger + ) { var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-resolution_"); try { var topLevelPackagesNames = packages.Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); - var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); - var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["build", tempProjectPath, "/t:_ReportDependencies"], tempDirectory.FullName, experimentsManager); - ThrowOnUnauthenticatedFeed(stdout); - - if (exitCode == 0) + Dependency[] allDependencies; + if (experimentsManager.UseDirectDiscovery) { - ImmutableArray tfms = [targetFramework]; - var lines = stdout.Split('\n').Select(line => line.Trim()); - var pattern = PackagePattern(); - var allDependencies = lines - .Select(line => pattern.Match(line)) - .Where(match => match.Success) - .Select(match => - { - var PackageName = match.Groups["PackageName"].Value; - var isTransitive = !topLevelPackagesNames.Contains(PackageName); - return new Dependency(PackageName, match.Groups["PackageVersion"].Value, DependencyType.Unknown, TargetFrameworks: tfms, IsTransitive: isTransitive); - }) - .ToArray(); - - return allDependencies; + var projectDiscovery = await SdkProjectDiscovery.DiscoverAsync(repoRoot, tempDirectory.FullName, tempProjectPath, experimentsManager, logger); + allDependencies = projectDiscovery + .Where(p => p.FilePath == Path.GetFileName(tempProjectPath)) + .FirstOrDefault() + ?.Dependencies.ToArray() ?? []; } else { - logger?.Warn($"dotnet build in {nameof(GetAllPackageDependenciesAsync)} failed. STDOUT: {stdout} STDERR: {stderr}"); - return []; + var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["build", tempProjectPath, "/t:_ReportDependencies"], tempDirectory.FullName, experimentsManager); + ThrowOnUnauthenticatedFeed(stdout); + + if (exitCode == 0) + { + ImmutableArray tfms = [targetFramework]; + var lines = stdout.Split('\n').Select(line => line.Trim()); + var pattern = PackagePattern(); + allDependencies = lines + .Select(line => pattern.Match(line)) + .Where(match => match.Success) + .Select(match => + { + var PackageName = match.Groups["PackageName"].Value; + var isTransitive = !topLevelPackagesNames.Contains(PackageName); + return new Dependency(PackageName, match.Groups["PackageVersion"].Value, DependencyType.Unknown, TargetFrameworks: tfms, IsTransitive: isTransitive); + }) + .ToArray(); + } + else + { + logger?.Warn($"dotnet build in {nameof(GetAllPackageDependenciesAsync)} failed. STDOUT: {stdout} STDERR: {stderr}"); + allDependencies = []; + } } + + return allDependencies; } finally { diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs index fb29c8e751..9fb12652d2 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs @@ -9,7 +9,7 @@ internal static async Task DownloadNuGetPackagesAsync(string repoRoot, str var tempDirectory = Directory.CreateTempSubdirectory("msbuild_sdk_restore_"); try { - var tempProjectPath = await MSBuildHelper.CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, "netstandard2.0", packages, logger, usePackageDownload: true); + var tempProjectPath = await MSBuildHelper.CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, "netstandard2.0", packages, experimentsManager, logger, usePackageDownload: true); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); return exitCode == 0;