Make use of a retry mechanism in the CommandRunner for retrying when an exception occurs

test/NuGet.Clients.FuncTests/NuGet.CommandLine.FuncTest/CommandRunnerTest.cs

@@ -192,7 +192,7 @@ public void Run_TimesOut()
 
             Stopwatch stopwatch = Stopwatch.StartNew();
 
-            new Action(() => CommandRunner.Run(fileName, arguments: args, timeOutInMilliseconds: 1000, testOutputHelper: _testOutputHelper))
+            new Action(() => CommandRunner.Run(fileName, arguments: args, timeOutInMilliseconds: 1000, testOutputHelper: _testOutputHelper, timeoutRetryCount: 0))
                 .Should().Throw<TimeoutException>();
 
             stopwatch.Stop();

test/NuGet.Clients.FuncTests/NuGet.CommandLine.FuncTest/RetryRunnerTest.cs

@@ -0,0 +1,123 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Xunit;
+using Xunit.Abstractions;
+using FluentAssertions;
+using NuGet.Test.Utility;
+
+namespace NuGet.CommandLine.FuncTest
+{
+    public class RetryRunnerTest
+    {
+        private readonly ITestOutputHelper _output;
+
+        public RetryRunnerTest(ITestOutputHelper output)
+        {
+            _output = output;
+        }
+
+        [Fact]
+        public void RunWithRetries_WhenNoException_ShouldReturnResult()
+        {
+            // Arrange
+            int maxRetries = 3;
+            int runCount = 0;
+            Func<int> func = () =>
+            {
+                runCount++;
+                return 42;
+            };
+
+            // Act
+            int result = RetryRunner.RunWithRetries<int, Exception>(func, maxRetries, _output);
+
+            // Assert
+            result.Should().Be(42);
+            runCount.Should().Be(1);
+        }
+
+        [Fact]
+        public void RunWithRetries_OnException_ShouldRetry()
+        {
+            // Arrange
+            int maxRetries = 1;
+            int runCount = 0;
+            Func<int> func = () =>
+            {
+                runCount++;
+                if (runCount < maxRetries + 1)
+                {
+                    throw new InvalidOperationException("Simulated exception");
+                }
+                return 42;
+            };
+
+            // Act
+            int result = RetryRunner.RunWithRetries<int, InvalidOperationException>(func, maxRetries, _output);
+
+            // Assert
+            result.Should().Be(42);
+            runCount.Should().Be(2); // Includes initial attempt
+        }
+
+        [Fact]
+        public void RunWithRetries_OnSuccess_ShouldNotRetry()
+        {
+            // Arrange
+            int maxRetries = 1;
+            int runCount = 0;
+            Func<int> func = () =>
+            {
+                runCount++;
+                if (runCount < maxRetries + 1)
+                {
+                    throw new InvalidOperationException("Simulated exception");
+                }
+                return 42;
+            };
+
+            // Act
+            int result = RetryRunner.RunWithRetries<int, InvalidOperationException>(func, maxRetries, _output);
+
+            // Assert
+            result.Should().Be(42);
+            runCount.Should().Be(2); // Only initial attempt
+        }
+
+        [Fact]
+        public void RunWithRetries_WhenMaxRetriesIsZero_ShouldNotRetry()
+        {
+            // Arrange
+            int maxRetries = 0;
+            int runCount = 0;
+            Func<int> func = () =>
+            {
+                runCount++;
+                throw new InvalidOperationException("Simulated exception");
+            };
+
+            // Act and Assert
+            Assert.Throws<InvalidOperationException>(() => RetryRunner.RunWithRetries<int, Exception>(func, maxRetries, _output));
+            runCount.Should().Be(1);
+        }
+
+        [Fact]
+        public void RunWithRetries_NonSpecifiedException_ShouldThrow()
+        {
+            // Arrange
+            int runCount = 0;
+            Func<int> func = () =>
+            {
+                runCount++;
+                throw new ArgumentException("Simulated exception");
+            };
+
+            // Act & Assert
+            Action act = () => RetryRunner.RunWithRetries<int, InvalidOperationException>(func);
+            act.Should().Throw<ArgumentException>(); // Should not catch and retry ArgumentException
+            runCount.Should().Be(1);
+        }
+    }
+}

test/TestUtilities/Test.Utility/CommandRunner.cs

@@ -25,106 +25,112 @@ public class CommandRunner
         /// <param name="inputAction">An optional <see cref="Action{T}" /> to invoke against the executables input stream.</param>
         /// <param name="environmentVariables">An optional <see cref="Dictionary{TKey, TValue}" /> containing environment variables to specify when running the executable.</param>
         /// <param name="testOutputHelper">An optional <see cref="ITestOutputHelper" /> to write output to.</param>
+        /// <param name="timeoutRetryCount">An optional number of times to retry running the command if it times out. Defaults to 1.</param>
         /// <returns>A <see cref="CommandRunnerResult" /> containing details about the result of the running the executable including the exit code and console output.</returns>
-        public static CommandRunnerResult Run(string filename, string workingDirectory = null, string arguments = null, int timeOutInMilliseconds = 60000, Action<StreamWriter> inputAction = null, IDictionary<string, string> environmentVariables = null, ITestOutputHelper testOutputHelper = null)
+        public static CommandRunnerResult Run(string filename, string workingDirectory = null, string arguments = null, int timeOutInMilliseconds = 60000, Action<StreamWriter> inputAction = null, IDictionary<string, string> environmentVariables = null, ITestOutputHelper testOutputHelper = null, int timeoutRetryCount = 1)
         {
-            StringBuilder output = new();
-            StringBuilder error = new();
-            int exitCode = 0;
-
-            using (Process process = new()
-            {
-                EnableRaisingEvents = true,
-                StartInfo = new ProcessStartInfo(Path.GetFullPath(filename), arguments)
-                {
-                    WorkingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory),
-                    UseShellExecute = false,
-                    RedirectStandardError = true,
-                    RedirectStandardOutput = true,
-                    RedirectStandardInput = true,
-                    CreateNoWindow = true,
-                },
-            })
+            return RetryRunner.RunWithRetries<CommandRunnerResult, TimeoutException>(() =>
             {
-                process.StartInfo.Environment["MSBUILDDISABLENODEREUSE"] = "1";
-                process.StartInfo.Environment["NUGET_SHOW_STACK"] = bool.TrueString;
-                process.StartInfo.Environment["NuGetTestModeEnabled"] = bool.TrueString;
-                process.StartInfo.Environment["UseSharedCompilation"] = bool.FalseString;
+                StringBuilder output = new();
+                StringBuilder error = new();
+                int exitCode = 0;
 
-                if (environmentVariables != null)
+                using (Process process = new()
                 {
-                    foreach (var pair in environmentVariables)
+                    EnableRaisingEvents = true,
+                    StartInfo = new ProcessStartInfo(Path.GetFullPath(filename), arguments)
                     {
-                        process.StartInfo.EnvironmentVariables[pair.Key] = pair.Value;
+                        WorkingDirectory = Path.GetFullPath(workingDirectory ?? Environment.CurrentDirectory),
+                        UseShellExecute = false,
+                        RedirectStandardError = true,
+                        RedirectStandardOutput = true,
+                        RedirectStandardInput = true,
+                        CreateNoWindow = true,
+                    },
+                })
+                {
+                    process.StartInfo.Environment["MSBUILDDISABLENODEREUSE"] = "1";
+                    process.StartInfo.Environment["NUGET_SHOW_STACK"] = bool.TrueString;
+                    process.StartInfo.Environment["NuGetTestModeEnabled"] = bool.TrueString;
+                    process.StartInfo.Environment["UseSharedCompilation"] = bool.FalseString;
+
+                    if (environmentVariables != null)
+                    {
+                        foreach (var pair in environmentVariables)
+                        {
+                            process.StartInfo.EnvironmentVariables[pair.Key] = pair.Value;
+                        }
                     }
-                }
 
-                process.OutputDataReceived += OnOutputDataReceived;
-                process.ErrorDataReceived += OnErrorDataReceived;
+                    process.OutputDataReceived += OnOutputDataReceived;
+                    process.ErrorDataReceived += OnErrorDataReceived;
 
-                testOutputHelper?.WriteLine($"> {process.StartInfo.FileName} {process.StartInfo.Arguments}");
+                    testOutputHelper?.WriteLine($"> {process.StartInfo.FileName} {process.StartInfo.Arguments}");
 
-                Stopwatch stopwatch = Stopwatch.StartNew();
+                    Stopwatch stopwatch = Stopwatch.StartNew();
 
-                process.Start();
+                    process.Start();
 
-                process.BeginOutputReadLine();
-                process.BeginErrorReadLine();
+                    process.BeginOutputReadLine();
+                    process.BeginErrorReadLine();
 
-                inputAction?.Invoke(process.StandardInput);
+                    inputAction?.Invoke(process.StandardInput);
 
-                process.StandardInput.Close();
+                    process.StandardInput.Close();
 
-                if (!process.WaitForExit(timeOutInMilliseconds))
-                {
-                    if (!process.HasExited)
+                    if (!process.WaitForExit(timeOutInMilliseconds))
                     {
-                        process.Kill();
-                    }
+                        if (!process.HasExited)
+                        {
+                            process.Kill();
+                        }
 
-                    throw new TimeoutException($"{process.StartInfo.FileName} {process.StartInfo.Arguments} timed out after {stopwatch.Elapsed.TotalSeconds:N2} seconds");
-                }
+                        throw new TimeoutException($"{process.StartInfo.FileName} {process.StartInfo.Arguments} timed out after {stopwatch.Elapsed.TotalSeconds:N2} seconds");
+                    }
 
-                // The application that is processing the asynchronous output should call the WaitForExit method to ensure that the output buffer has been flushed.
-                process.WaitForExit();
+                    // The application that is processing the asynchronous output should call the WaitForExit method to ensure that the output buffer has been flushed.
+                    process.WaitForExit();
 
-                stopwatch.Stop();
-                testOutputHelper?.WriteLine($"└ Completed in {stopwatch.Elapsed.TotalSeconds:N2}s");
+                    stopwatch.Stop();
+                    testOutputHelper?.WriteLine($"└ Completed in {stopwatch.Elapsed.TotalSeconds:N2}s");
 
-                process.OutputDataReceived -= OnOutputDataReceived;
-                process.ErrorDataReceived -= OnErrorDataReceived;
+                    process.OutputDataReceived -= OnOutputDataReceived;
+                    process.ErrorDataReceived -= OnErrorDataReceived;
 
-                testOutputHelper?.WriteLine(string.Empty);
-                exitCode = process.ExitCode;
-            }
+                    testOutputHelper?.WriteLine(string.Empty);
+                    exitCode = process.ExitCode;
+                }
 
-            return new CommandRunnerResult(exitCode, output.ToString(), error.ToString());
+                return new CommandRunnerResult(exitCode, output.ToString(), error.ToString());
 
-            void OnOutputDataReceived(object sender, DataReceivedEventArgs args)
-            {
-                if (args?.Data != null)
+                void OnOutputDataReceived(object sender, DataReceivedEventArgs args)
                 {
-                    testOutputHelper?.WriteLine($"│  {args.Data}");
-
-                    lock (output)
+                    if (args?.Data != null)
                     {
-                        output.AppendLine(args.Data);
+                        testOutputHelper?.WriteLine($"│  {args.Data}");
+
+                        lock (output)
+                        {
+                            output.AppendLine(args.Data);
+                        }
                     }
                 }
-            }
 
-            void OnErrorDataReceived(object sender, DataReceivedEventArgs args)
-            {
-                if (args?.Data != null)
+                void OnErrorDataReceived(object sender, DataReceivedEventArgs args)
                 {
-                    testOutputHelper?.WriteLine($"│  {args.Data}");
-
-                    lock (error)
+                    if (args?.Data != null)
                     {
-                        error.AppendLine(args.Data);
+                        testOutputHelper?.WriteLine($"│  {args.Data}");
+
+                        lock (error)
+                        {
+                            error.AppendLine(args.Data);
+                        }
                     }
                 }
-            }
+            },
+            timeoutRetryCount,
+            testOutputHelper);
         }
     }
 }

test/TestUtilities/Test.Utility/RetryRunner.cs

@@ -0,0 +1,37 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Xunit.Abstractions;
+
+namespace NuGet.Test.Utility
+{
+    internal class RetryRunner
+    {
+        public static T RunWithRetries<T, E>(Func<T> func, int maxRetries = 1, ITestOutputHelper logger = null) where E : Exception
+        {
+            {
+                int retryCount = 0;
+
+                while (true)
+                {
+                    try
+                    {
+                        return func();
+                    }
+                    catch (E exception)
+                    {
+                        if (retryCount >= maxRetries)
+                        {
+                            throw exception;
+                        }
+
+                        retryCount++;
+                        logger?.WriteLine($"Encountered exception during run attempt #{retryCount}: {exception.Message}");
+                        logger?.WriteLine($"Retrying {retryCount} of {maxRetries}");
+                    }
+                }
+            }
+        }
+    }
+}