diff --git a/nuget/helpers/lib/NuGetUpdater/.editorconfig b/nuget/helpers/lib/NuGetUpdater/.editorconfig index c5fa72bccd..55f434de48 100644 --- a/nuget/helpers/lib/NuGetUpdater/.editorconfig +++ b/nuget/helpers/lib/NuGetUpdater/.editorconfig @@ -20,6 +20,7 @@ tab_width = 4 # New line preferences insert_final_newline = true +end_of_line = lf #### .NET Coding Conventions #### [*.{cs,vb}] diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs index bf75154bcd..944202c9f1 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs @@ -169,6 +169,98 @@ await RunAsync( ); } + [Fact] + public async Task PrivateSourceAuthenticationFailureIsForwaredToApiHandler() + { + static (int, string) TestHttpHandler(string uriString) + { + var uri = new Uri(uriString, UriKind.Absolute); + var baseUrl = $"{uri.Scheme}://{uri.Host}:{uri.Port}"; + return uri.PathAndQuery switch + { + // initial request is good + "/index.json" => (200, $$""" + { + "version": "3.0.0", + "resources": [ + { + "@id": "{{baseUrl}}/download", + "@type": "PackageBaseAddress/3.0.0" + }, + { + "@id": "{{baseUrl}}/query", + "@type": "SearchQueryService" + }, + { + "@id": "{{baseUrl}}/registrations", + "@type": "RegistrationsBaseUrl" + } + ] + } + """), + // all other requests are unauthorized + _ => (401, "{}"), + }; + } + using var http = TestHttpServer.CreateTestStringServer(TestHttpHandler); + await RunAsync( + packages: + [ + ], + job: new Job() + { + PackageManager = "nuget", + Source = new() + { + Provider = "github", + Repo = "test/repo", + Directory = "/", + }, + AllowedUpdates = + [ + new() { UpdateType = "all" } + ] + }, + files: + [ + ("NuGet.Config", $""" + + + + + + + """), + ("project.csproj", """ + + + net8.0 + + + + + + """) + ], + expectedResult: new RunResult() + { + Base64DependencyFiles = [], + BaseCommitSha = "TEST-COMMIT-SHA", + }, + expectedApiMessages: + [ + new PrivateSourceAuthenticationFailure() + { + Details = $"({http.BaseUrl.TrimEnd('/')}/index.json)" + }, + new MarkAsProcessed() + { + BaseCommitSha = "TEST-COMMIT-SHA", + } + ] + ); + } + private static async Task RunAsync(Job job, TestFile[] files, RunResult expectedResult, object[] expectedApiMessages, MockNuGetPackage[]? packages = null) { // arrange diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs index 52954bf637..07d3000dbd 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs @@ -9,27 +9,33 @@ internal class TestApiHandler : IApiHandler public IEnumerable<(Type Type, object Object)> ReceivedMessages => _receivedMessages; + public Task RecordUpdateJobError(JobErrorBase error) + { + _receivedMessages.Add((error.GetType(), error)); + return Task.CompletedTask; + } + public Task UpdateDependencyList(UpdatedDependencyList updatedDependencyList) { - _receivedMessages.Add((typeof(UpdatedDependencyList), updatedDependencyList)); + _receivedMessages.Add((updatedDependencyList.GetType(), updatedDependencyList)); return Task.CompletedTask; } public Task IncrementMetric(IncrementMetric incrementMetric) { - _receivedMessages.Add((typeof(IncrementMetric), incrementMetric)); + _receivedMessages.Add((incrementMetric.GetType(), incrementMetric)); return Task.CompletedTask; } public Task CreatePullRequest(CreatePullRequest createPullRequest) { - _receivedMessages.Add((typeof(CreatePullRequest), createPullRequest)); + _receivedMessages.Add((createPullRequest.GetType(), createPullRequest)); return Task.CompletedTask; } public Task MarkAsProcessed(MarkAsProcessed markAsProcessed) { - _receivedMessages.Add((typeof(MarkAsProcessed), markAsProcessed)); + _receivedMessages.Add((markAsProcessed.GetType(), markAsProcessed)); return Task.CompletedTask; } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs index a9f5eac33c..8b0a91b4c8 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs @@ -257,14 +257,6 @@ public static async Task MockNuGetPackagesInDirectory(MockNuGetPackage[]? packag package.WriteToDirectory(localFeedPath); } - // override various nuget locations - foreach (var envName in new[] { "NUGET_PACKAGES", "NUGET_HTTP_CACHE_PATH", "NUGET_SCRATCH", "NUGET_PLUGINS_CACHE_PATH" }) - { - string dir = Path.Join(temporaryDirectory, envName); - Directory.CreateDirectory(dir); - Environment.SetEnvironmentVariable(envName, dir); - } - // ensure only the test feed is used string relativeLocalFeedPath = Path.GetRelativePath(temporaryDirectory, localFeedPath); await File.WriteAllTextAsync(Path.Join(temporaryDirectory, "NuGet.Config"), $""" @@ -278,6 +270,14 @@ await File.WriteAllTextAsync(Path.Join(temporaryDirectory, "NuGet.Config"), $""" """ ); } + + // override various nuget locations + foreach (var envName in new[] { "NUGET_PACKAGES", "NUGET_HTTP_CACHE_PATH", "NUGET_SCRATCH", "NUGET_PLUGINS_CACHE_PATH" }) + { + string dir = Path.Join(temporaryDirectory, envName); + Directory.CreateDirectory(dir); + Environment.SetEnvironmentVariable(envName, dir); + } } protected static async Task RunUpdate(TestFile[] files, Func action) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs index d959ce52e6..5d86b2c715 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs @@ -31,9 +31,29 @@ public AnalyzeWorker(Logger logger) public async Task RunAsync(string repoRoot, string discoveryPath, string dependencyPath, string analysisDirectory) { + AnalysisResult analysisResult; var discovery = await DeserializeJsonFileAsync(discoveryPath, nameof(WorkspaceDiscoveryResult)); var dependencyInfo = await DeserializeJsonFileAsync(dependencyPath, nameof(DependencyInfo)); - var analysisResult = await RunAsync(repoRoot, discovery, dependencyInfo); + + try + { + analysisResult = await RunAsync(repoRoot, discovery, dependencyInfo); + } + catch (HttpRequestException ex) + when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) + { + var localPath = PathHelper.JoinPath(repoRoot, discovery.Path); + var nugetContext = new NuGetContext(localPath); + analysisResult = new AnalysisResult + { + ErrorType = ErrorType.AuthenticationFailure, + ErrorDetails = "(" + string.Join("|", nugetContext.PackageSources.Select(s => s.Source)) + ")", + UpdatedVersion = string.Empty, + CanUpdate = false, + UpdatedDependencies = [], + }; + } + await WriteResultsAsync(analysisDirectory, dependencyInfo.Name, analysisResult, _logger); } @@ -68,100 +88,84 @@ public async Task RunAsync(string repoRoot, WorkspaceDiscoveryRe var isUpdateNecessary = isProjectUpdateNecessary || dotnetToolsHasDependency || globalJsonHasDependency; using var nugetContext = new NuGetContext(startingDirectory); AnalysisResult analysisResult; - try + if (isUpdateNecessary) { - if (isUpdateNecessary) + _logger.Log($" Determining multi-dependency property."); + var multiDependencies = DetermineMultiDependencyDetails( + discovery, + dependencyInfo.Name, + propertyBasedDependencies); + + usesMultiDependencyProperty = multiDependencies.Any(md => md.DependencyNames.Count > 1); + var dependenciesToUpdate = usesMultiDependencyProperty + ? multiDependencies + .SelectMany(md => md.DependencyNames) + .ToImmutableHashSet(StringComparer.OrdinalIgnoreCase) + : [dependencyInfo.Name]; + var applicableTargetFrameworks = usesMultiDependencyProperty + ? multiDependencies + .SelectMany(md => md.TargetFrameworks) + .ToImmutableHashSet(StringComparer.OrdinalIgnoreCase) + .Select(NuGetFramework.Parse) + .ToImmutableArray() + : projectFrameworks; + + _logger.Log($" Finding updated version."); + updatedVersion = await FindUpdatedVersionAsync( + startingDirectory, + dependencyInfo, + dependenciesToUpdate, + applicableTargetFrameworks, + nugetContext, + _logger, + CancellationToken.None); + + _logger.Log($" Finding updated peer dependencies."); + if (updatedVersion is null) + { + updatedDependencies = []; + } + else if (isProjectUpdateNecessary) { - _logger.Log($" Determining multi-dependency property."); - var multiDependencies = DetermineMultiDependencyDetails( + updatedDependencies = await FindUpdatedDependenciesAsync( + repoRoot, discovery, - dependencyInfo.Name, - propertyBasedDependencies); - - usesMultiDependencyProperty = multiDependencies.Any(md => md.DependencyNames.Count > 1); - var dependenciesToUpdate = usesMultiDependencyProperty - ? multiDependencies - .SelectMany(md => md.DependencyNames) - .ToImmutableHashSet(StringComparer.OrdinalIgnoreCase) - : [dependencyInfo.Name]; - var applicableTargetFrameworks = usesMultiDependencyProperty - ? multiDependencies - .SelectMany(md => md.TargetFrameworks) - .ToImmutableHashSet(StringComparer.OrdinalIgnoreCase) - .Select(NuGetFramework.Parse) - .ToImmutableArray() - : projectFrameworks; - - _logger.Log($" Finding updated version."); - updatedVersion = await FindUpdatedVersionAsync( - startingDirectory, - dependencyInfo, dependenciesToUpdate, - applicableTargetFrameworks, + updatedVersion, nugetContext, _logger, CancellationToken.None); - - _logger.Log($" Finding updated peer dependencies."); - if (updatedVersion is null) - { - updatedDependencies = []; - } - else if (isProjectUpdateNecessary) - { - updatedDependencies = await FindUpdatedDependenciesAsync( - repoRoot, - discovery, - dependenciesToUpdate, - updatedVersion, - nugetContext, - _logger, - CancellationToken.None); - } - else if (dotnetToolsHasDependency) - { - var infoUrl = await nugetContext.GetPackageInfoUrlAsync(dependencyInfo.Name, updatedVersion.ToNormalizedString(), CancellationToken.None); - updatedDependencies = [new Dependency(dependencyInfo.Name, updatedVersion.ToNormalizedString(), DependencyType.DotNetTool, IsDirect: true, InfoUrl: infoUrl)]; - } - else if (globalJsonHasDependency) - { - var infoUrl = await nugetContext.GetPackageInfoUrlAsync(dependencyInfo.Name, updatedVersion.ToNormalizedString(), CancellationToken.None); - updatedDependencies = [new Dependency(dependencyInfo.Name, updatedVersion.ToNormalizedString(), DependencyType.MSBuildSdk, IsDirect: true, InfoUrl: infoUrl)]; - } - else - { - throw new InvalidOperationException("Unreachable."); - } - - //TODO: At this point we should add the peer dependencies to a queue where - // we will analyze them one by one to see if they themselves are part of a - // multi-dependency property. Basically looping this if-body until we have - // emptied the queue and have a complete list of updated dependencies. We - // should track the dependenciesToUpdate as they have already been analyzed. } - - analysisResult = new AnalysisResult + else if (dotnetToolsHasDependency) { - UpdatedVersion = updatedVersion?.ToNormalizedString() ?? dependencyInfo.Version, - CanUpdate = updatedVersion is not null, - VersionComesFromMultiDependencyProperty = usesMultiDependencyProperty, - UpdatedDependencies = updatedDependencies, - }; - } - catch (HttpRequestException ex) - when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) - { - // TODO: consolidate this error handling between AnalyzeWorker, DiscoveryWorker, and UpdateWorker - analysisResult = new AnalysisResult + var infoUrl = await nugetContext.GetPackageInfoUrlAsync(dependencyInfo.Name, updatedVersion.ToNormalizedString(), CancellationToken.None); + updatedDependencies = [new Dependency(dependencyInfo.Name, updatedVersion.ToNormalizedString(), DependencyType.DotNetTool, IsDirect: true, InfoUrl: infoUrl)]; + } + else if (globalJsonHasDependency) { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "(" + string.Join("|", nugetContext.PackageSources.Select(s => s.Source)) + ")", - UpdatedVersion = string.Empty, - CanUpdate = false, - UpdatedDependencies = [], - }; + var infoUrl = await nugetContext.GetPackageInfoUrlAsync(dependencyInfo.Name, updatedVersion.ToNormalizedString(), CancellationToken.None); + updatedDependencies = [new Dependency(dependencyInfo.Name, updatedVersion.ToNormalizedString(), DependencyType.MSBuildSdk, IsDirect: true, InfoUrl: infoUrl)]; + } + else + { + throw new InvalidOperationException("Unreachable."); + } + + //TODO: At this point we should add the peer dependencies to a queue where + // we will analyze them one by one to see if they themselves are part of a + // multi-dependency property. Basically looping this if-body until we have + // emptied the queue and have a complete list of updated dependencies. We + // should track the dependenciesToUpdate as they have already been analyzed. } + analysisResult = new AnalysisResult + { + UpdatedVersion = updatedVersion?.ToNormalizedString() ?? dependencyInfo.Version, + CanUpdate = updatedVersion is not null, + VersionComesFromMultiDependencyProperty = usesMultiDependencyProperty, + UpdatedDependencies = updatedDependencies, + }; + _logger.Log($"Analysis complete."); return analysisResult; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs index cb05172112..c1027f46cd 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs @@ -30,7 +30,29 @@ public DiscoveryWorker(Logger logger) _logger = logger; } - public async Task RunAsync(string repoRootPath, string workspacePath) + public async Task RunAsync(string repoRootPath, string workspacePath, string outputPath) + { + WorkspaceDiscoveryResult result; + try + { + result = await RunAsync(repoRootPath, workspacePath); + } + catch (HttpRequestException ex) + when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) + { + result = new WorkspaceDiscoveryResult + { + ErrorType = ErrorType.AuthenticationFailure, + ErrorDetails = "(" + string.Join("|", NuGetContext.GetPackageSourceUrls(PathHelper.JoinPath(repoRootPath, workspacePath))) + ")", + Path = workspacePath, + Projects = [], + }; + } + + await WriteResultsAsync(repoRootPath, outputPath, result); + } + + internal async Task RunAsync(string repoRootPath, string workspacePath) { MSBuildHelper.RegisterMSBuild(Environment.CurrentDirectory, repoRootPath); @@ -51,69 +73,48 @@ public async Task RunAsync(string repoRootPath, string ImmutableArray projectResults = []; WorkspaceDiscoveryResult result; - try + if (Directory.Exists(workspacePath)) { - if (Directory.Exists(workspacePath)) - { - _logger.Log($"Discovering build files in workspace [{workspacePath}]."); + _logger.Log($"Discovering build files in workspace [{workspacePath}]."); - dotNetToolsJsonDiscovery = DotNetToolsJsonDiscovery.Discover(repoRootPath, workspacePath, _logger); - globalJsonDiscovery = GlobalJsonDiscovery.Discover(repoRootPath, workspacePath, _logger); + dotNetToolsJsonDiscovery = DotNetToolsJsonDiscovery.Discover(repoRootPath, workspacePath, _logger); + globalJsonDiscovery = GlobalJsonDiscovery.Discover(repoRootPath, workspacePath, _logger); - if (globalJsonDiscovery is not null) - { - await TryRestoreMSBuildSdksAsync(repoRootPath, workspacePath, globalJsonDiscovery.Dependencies, _logger); - } + if (globalJsonDiscovery is not null) + { + await TryRestoreMSBuildSdksAsync(repoRootPath, workspacePath, globalJsonDiscovery.Dependencies, _logger); + } - // this next line should throw or something - projectResults = await RunForDirectoryAsnyc(repoRootPath, workspacePath); + // this next line should throw or something + projectResults = await RunForDirectoryAsnyc(repoRootPath, workspacePath); - directoryPackagesPropsDiscovery = DirectoryPackagesPropsDiscovery.Discover(repoRootPath, workspacePath, projectResults, _logger); + directoryPackagesPropsDiscovery = DirectoryPackagesPropsDiscovery.Discover(repoRootPath, workspacePath, projectResults, _logger); - if (directoryPackagesPropsDiscovery is not null) - { - projectResults = projectResults.Remove(projectResults.First(p => p.FilePath.Equals(directoryPackagesPropsDiscovery.FilePath, StringComparison.OrdinalIgnoreCase))); - } - } - else + if (directoryPackagesPropsDiscovery is not null) { - _logger.Log($"Workspace path [{workspacePath}] does not exist."); + projectResults = projectResults.Remove(projectResults.First(p => p.FilePath.Equals(directoryPackagesPropsDiscovery.FilePath, StringComparison.OrdinalIgnoreCase))); } - - result = new WorkspaceDiscoveryResult - { - Path = initialWorkspacePath, - DotNetToolsJson = dotNetToolsJsonDiscovery, - GlobalJson = globalJsonDiscovery, - DirectoryPackagesProps = directoryPackagesPropsDiscovery, - Projects = projectResults.OrderBy(p => p.FilePath).ToImmutableArray(), - }; } - catch (HttpRequestException ex) - when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) + else { - // TODO: consolidate this error handling between AnalyzeWorker, DiscoveryWorker, and UpdateWorker - result = new WorkspaceDiscoveryResult - { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "(" + string.Join("|", NuGetContext.GetPackageSourceUrls(workspacePath)) + ")", - Path = initialWorkspacePath, - Projects = [], - }; + _logger.Log($"Workspace path [{workspacePath}] does not exist."); } + result = new WorkspaceDiscoveryResult + { + Path = initialWorkspacePath, + DotNetToolsJson = dotNetToolsJsonDiscovery, + GlobalJson = globalJsonDiscovery, + DirectoryPackagesProps = directoryPackagesPropsDiscovery, + Projects = projectResults.OrderBy(p => p.FilePath).ToImmutableArray(), + }; + _logger.Log("Discovery complete."); _processedProjectPaths.Clear(); return result; } - public async Task RunAsync(string repoRootPath, string workspacePath, string outputPath) - { - var result = await RunAsync(repoRootPath, workspacePath); - await WriteResultsAsync(repoRootPath, outputPath, result); - } - /// /// Restores MSBuild SDKs from the given dependencies. /// diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs new file mode 100644 index 0000000000..8188266a59 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs @@ -0,0 +1,6 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public record DependencyFileNotFound : JobErrorBase +{ + public override string Type => "dependency_file_not_found"; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs new file mode 100644 index 0000000000..1b115a04bd --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace NuGetUpdater.Core.Run.ApiModel; + +public abstract record JobErrorBase +{ + [JsonPropertyName("error-type")] + public abstract string Type { get; } + [JsonPropertyName("error-details")] + public required string Details { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/PrivateSourceAuthenticationFailure.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/PrivateSourceAuthenticationFailure.cs new file mode 100644 index 0000000000..616cb9bd62 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/PrivateSourceAuthenticationFailure.cs @@ -0,0 +1,6 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public record PrivateSourceAuthenticationFailure : JobErrorBase +{ + public override string Type => "private_source_authentication_failure"; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/UnknownError.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/UnknownError.cs new file mode 100644 index 0000000000..7ab4b455fc --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/UnknownError.cs @@ -0,0 +1,6 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public record UnknownError : JobErrorBase +{ + public override string Type => "unknown_error"; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs index 40d25203d1..bcad298776 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs @@ -25,6 +25,11 @@ public HttpApiHandler(string apiUrl, string jobId) _jobId = jobId; } + public async Task RecordUpdateJobError(JobErrorBase error) + { + await PostAsJson("record_update_job_error", error); + } + public async Task UpdateDependencyList(UpdatedDependencyList updatedDependencyList) { await PostAsJson("update_dependency_list", updatedDependencyList); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs index 0586795521..ce646d97a7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs @@ -4,6 +4,7 @@ namespace NuGetUpdater.Core.Run; public interface IApiHandler { + Task RecordUpdateJobError(JobErrorBase error); Task UpdateDependencyList(UpdatedDependencyList updatedDependencyList); Task IncrementMetric(IncrementMetric incrementMetric); Task CreatePullRequest(CreatePullRequest createPullRequest); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs index b3b8c4fcc5..fe5eb092c2 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -35,25 +36,73 @@ public async Task RunAsync(FileInfo jobFilePath, DirectoryInfo repoContentsPath, await File.WriteAllTextAsync(outputFilePath.FullName, resultJson); } - public async Task RunAsync(Job job, DirectoryInfo repoContentsPath, string baseCommitSha) + public Task RunAsync(Job job, DirectoryInfo repoContentsPath, string baseCommitSha) { - MSBuildHelper.RegisterMSBuild(repoContentsPath.FullName, repoContentsPath.FullName); + return RunWithErrorHandlingAsync(job, repoContentsPath, baseCommitSha); + } + + private async Task RunWithErrorHandlingAsync(Job job, DirectoryInfo repoContentsPath, string baseCommitSha) + { + JobErrorBase? error = null; + string[] lastUsedPackageSourceUrls = []; // used for error reporting below + var runResult = new RunResult() + { + Base64DependencyFiles = [], + BaseCommitSha = baseCommitSha, + }; - var allDependencyFiles = new Dictionary(); - foreach (var directory in job.GetAllDirectories()) + try { - var result = await RunForDirectory(job, repoContentsPath, directory, baseCommitSha); - foreach (var dependencyFile in result.Base64DependencyFiles) + MSBuildHelper.RegisterMSBuild(repoContentsPath.FullName, repoContentsPath.FullName); + + var allDependencyFiles = new Dictionary(); + foreach (var directory in job.GetAllDirectories()) { - allDependencyFiles[dependencyFile.Name] = dependencyFile; + var localPath = PathHelper.JoinPath(repoContentsPath.FullName, directory); + lastUsedPackageSourceUrls = NuGetContext.GetPackageSourceUrls(localPath); + var result = await RunForDirectory(job, repoContentsPath, directory, baseCommitSha); + foreach (var dependencyFile in result.Base64DependencyFiles) + { + allDependencyFiles[dependencyFile.Name] = dependencyFile; + } } + + runResult = new RunResult() + { + Base64DependencyFiles = allDependencyFiles.Values.ToArray(), + BaseCommitSha = baseCommitSha, + }; + } + catch (HttpRequestException ex) + when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) + { + error = new PrivateSourceAuthenticationFailure() + { + Details = $"({string.Join("|", lastUsedPackageSourceUrls)})", + }; + } + catch (MissingFileException ex) + { + error = new DependencyFileNotFound() + { + Details = ex.FilePath, + }; + } + catch (Exception ex) + { + error = new UnknownError() + { + Details = ex.ToString(), + }; } - var runResult = new RunResult() + if (error is not null) { - Base64DependencyFiles = allDependencyFiles.Values.ToArray(), - BaseCommitSha = baseCommitSha, - }; + await _apiHandler.RecordUpdateJobError(error); + } + + await _apiHandler.MarkAsProcessed(new() { BaseCommitSha = baseCommitSha }); + return runResult; } @@ -61,7 +110,6 @@ private async Task RunForDirectory(Job job, DirectoryInfo repoContent { var discoveryWorker = new DiscoveryWorker(_logger); var discoveryResult = await discoveryWorker.RunAsync(repoContentsPath.FullName, repoDirectory); - // TODO: check discoveryResult.ErrorType _logger.Log("Discovery JSON content:"); _logger.Log(JsonSerializer.Serialize(discoveryResult, DiscoveryWorker.SerializerOptions)); @@ -123,7 +171,6 @@ await _apiHandler.IncrementMetric(new() }; var analysisResult = await analyzeWorker.RunAsync(repoContentsPath.FullName, discoveryResult, dependencyInfo); // TODO: log analysisResult - // TODO: check analysisResult.ErrorType if (analysisResult.CanUpdate) { // TODO: this is inefficient, but not likely causing a bottleneck @@ -153,7 +200,6 @@ await _apiHandler.IncrementMetric(new() var updateWorker = new UpdaterWorker(_logger); var dependencyFilePath = Path.Join(discoveryResult.Path, project.FilePath).NormalizePathToUnix(); var updateResult = await updateWorker.RunAsync(repoContentsPath.FullName, dependencyFilePath, dependency.Name, dependency.Version!, analysisResult.UpdatedVersion, isTransitive: false); - // TODO: check specific contents of result.ErrorType // TODO: need to report if anything was actually updated if (updateResult.ErrorType is null || updateResult.ErrorType == ErrorType.None) { @@ -206,7 +252,6 @@ await _apiHandler.IncrementMetric(new() // TODO: throw if no updates performed } - await _apiHandler.MarkAsProcessed(new() { BaseCommitSha = baseCommitSha }); var result = new RunResult() { Base64DependencyFiles = originalDependencyFileContents.Select(kvp => new DependencyFile() diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs index db0f3b7fd8..c857d34f1a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs @@ -25,57 +25,19 @@ public UpdaterWorker(Logger logger) public async Task RunAsync(string repoRootPath, string workspacePath, string dependencyName, string previousDependencyVersion, string newDependencyVersion, bool isTransitive, string? resultOutputPath = null) { - var result = await RunAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); - if (resultOutputPath is { }) - { - await WriteResultFile(result, resultOutputPath, _logger); - } - } - - public async Task RunAsync(string repoRootPath, string workspacePath, string dependencyName, string previousDependencyVersion, string newDependencyVersion, bool isTransitive) - { - MSBuildHelper.RegisterMSBuild(Environment.CurrentDirectory, repoRootPath); - UpdateOperationResult result; - - if (!Path.IsPathRooted(workspacePath) || !File.Exists(workspacePath)) - { - workspacePath = Path.GetFullPath(Path.Join(repoRootPath, workspacePath)); - } - + UpdateOperationResult result = new(); // assumed to be ok until proven otherwise try { - if (!isTransitive) - { - await DotNetToolsJsonUpdater.UpdateDependencyAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, _logger); - await GlobalJsonUpdater.UpdateDependencyAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, _logger); - } - - var extension = Path.GetExtension(workspacePath).ToLowerInvariant(); - switch (extension) - { - case ".sln": - await RunForSolutionAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); - break; - case ".proj": - await RunForProjFileAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); - break; - case ".csproj": - case ".fsproj": - case ".vbproj": - await RunForProjectAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); - break; - default: - _logger.Log($"File extension [{extension}] is not supported."); - break; - } - - result = new(); // all ok - _logger.Log("Update complete."); + result = await RunAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); } catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) { - // TODO: consolidate this error handling between AnalyzeWorker, DiscoveryWorker, and UpdateWorker + if (!Path.IsPathRooted(workspacePath) || !File.Exists(workspacePath)) + { + workspacePath = Path.GetFullPath(Path.Join(repoRootPath, workspacePath)); + } + result = new() { ErrorType = ErrorType.AuthenticationFailure, @@ -91,8 +53,50 @@ public async Task RunAsync(string repoRootPath, string wo }; } + if (resultOutputPath is { }) + { + await WriteResultFile(result, resultOutputPath, _logger); + } + } + + public async Task RunAsync(string repoRootPath, string workspacePath, string dependencyName, string previousDependencyVersion, string newDependencyVersion, bool isTransitive) + { + MSBuildHelper.RegisterMSBuild(Environment.CurrentDirectory, repoRootPath); + + if (!Path.IsPathRooted(workspacePath) || !File.Exists(workspacePath)) + { + workspacePath = Path.GetFullPath(Path.Join(repoRootPath, workspacePath)); + } + + if (!isTransitive) + { + await DotNetToolsJsonUpdater.UpdateDependencyAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, _logger); + await GlobalJsonUpdater.UpdateDependencyAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, _logger); + } + + var extension = Path.GetExtension(workspacePath).ToLowerInvariant(); + switch (extension) + { + case ".sln": + await RunForSolutionAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); + break; + case ".proj": + await RunForProjFileAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); + break; + case ".csproj": + case ".fsproj": + case ".vbproj": + await RunForProjectAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); + break; + default: + _logger.Log($"File extension [{extension}] is not supported."); + break; + } + + _logger.Log("Update complete."); + _processedProjectPaths.Clear(); - return result; + return new UpdateOperationResult(); } internal static async Task WriteResultFile(UpdateOperationResult result, string resultOutputPath, Logger logger)