[CI] Test summary reporter improvements (#9263)

* GenerateTestReport: add logging, add artifacts url
* Fix GenerateTestReport run on windows
* Update tools/GenerateTestSummary/TestSummaryGenerator.cs

Co-authored-by: Dan Moseley <danmose@microsoft.com>

* address review feedback from @ danmoseley

* Address review feedback from @ russkie

---------

Co-authored-by: Dan Moseley <danmose@microsoft.com>

Author: radical

.github/workflows/run-tests.yml

@@ -300,11 +300,8 @@ jobs:
         if: always()
         env:
           CI: false
-        run: >
-          test -d ${{ github.workspace }}/testresults &&
-          ${{ github.workspace }}/dotnet.sh
-          run
-          --project ${{ github.workspace }}/tools/GenerateTestSummary/GenerateTestSummary.csproj
-          --
-          ${{ github.workspace }}/testresults
-          -u ${{ steps.upload-logs.outputs.artifact-url }}
+        shell: pwsh
+        run: |
+          if (Test-Path "${{ github.workspace }}/testresults") {
+            & ${{ env.DOTNET_SCRIPT }} run --project "${{ github.workspace }}/tools/GenerateTestSummary/GenerateTestSummary.csproj" -- "${{ github.workspace }}/testresults" -u "${{ steps.upload-logs.outputs.artifact-url }}"
+          }

tools/GenerateTestSummary/Program.cs

@@ -27,11 +27,18 @@
     var dirPathOrTrxFilePath = result.GetValue<string>(dirPathOrTrxFilePathArgument);
     if (string.IsNullOrEmpty(dirPathOrTrxFilePath))
     {
-        Console.WriteLine("Please provide a directory path with trx files or a trx file path.");
+        Console.WriteLine("Error: Please provide a directory path with trx files or a trx file path.");
         return;
     }
 
     var combinedSummary = result.GetValue<bool>(combinedSummaryOption);
+    var url = result.GetValue<string>(urlOption);
+
+    if (combinedSummary && !string.IsNullOrEmpty(url))
+    {
+        Console.WriteLine("Error: --url option is not supported with --combined option.");
+        return;
+    }
 
     string report;
     if (combinedSummary)
@@ -43,15 +50,22 @@
         var reportBuilder = new StringBuilder();
         if (Directory.Exists(dirPathOrTrxFilePath))
         {
-            var trxFiles = Directory.EnumerateFiles(dirPathOrTrxFilePath, "*.trx", SearchOption.AllDirectories);
-            foreach (var trxFile in trxFiles)
+            var trxFiles = Directory.EnumerateFiles(dirPathOrTrxFilePath, "*.trx", SearchOption.AllDirectories).ToList();
+            if (trxFiles.Count == 0)
+            {
+                Console.WriteLine($"Warning: No trx files found in directory: {dirPathOrTrxFilePath}");
+            }
+            else
             {
-                TestSummaryGenerator.CreateSingleTestSummaryReport(trxFile, reportBuilder);
+                foreach (var trxFile in trxFiles)
+                {
+                    TestSummaryGenerator.CreateSingleTestSummaryReport(trxFile, reportBuilder, url);
+                }
             }
         }
         else
         {
-            TestSummaryGenerator.CreateSingleTestSummaryReport(dirPathOrTrxFilePath, reportBuilder, result.GetValue<string>(urlOption));
+            TestSummaryGenerator.CreateSingleTestSummaryReport(dirPathOrTrxFilePath, reportBuilder, url);
         }
 
         report = reportBuilder.ToString();
@@ -73,12 +87,11 @@
     if (Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true"
         && Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is string summaryPath)
     {
+        Console.WriteLine($"Detected GitHub Actions environment. Writing to {summaryPath}");
         File.WriteAllText(summaryPath, report);
     }
-    else
-    {
-        Console.WriteLine(report);
-    }
+
+    Console.WriteLine(report);
 });
 
 return rootCommand.Parse(args).Invoke();

tools/GenerateTestSummary/TestSummaryGenerator.cs

@@ -66,7 +66,7 @@ public static string CreateCombinedTestSummaryReport(string basePath)
                             ? "lin"
                             : filePath.Contains("macos-")
                                 ? "mac"
-                                : "os?";
+                                : throw new InvalidOperationException($"Could not determine OS from file path: {filePath}");
 
             tableBuilder.AppendLine(CultureInfo.InvariantCulture, $"| {(failed > 0 ? "❌" : "✅")} [{os}] {GetTestTitle(filePath)} | {passed} | {failed} | {skipped} | {total} |");
         }
@@ -84,7 +84,7 @@ public static string CreateCombinedTestSummaryReport(string basePath)
         return overallTableBuilder.ToString();
     }
 
-    public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuilder reportBuilder, string? url = null)
+    public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuilder reportBuilder, string? url)
     {
         if (!File.Exists(trxFilePath))
         {
@@ -109,21 +109,27 @@ public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuild
         var failed = counters.Failed;
         if (failed == 0)
         {
+            Console.WriteLine($"No failed tests in {trxFilePath}");
             return;
         }
 
         var total = counters.Total;
         var passed = counters.Passed;
         var skipped = counters.NotExecuted;
 
-        reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"### {GetTestTitle(trxFilePath)}");
+        var title = string.IsNullOrEmpty(url)
+            ? GetTestTitle(trxFilePath)
+            : $"{GetTestTitle(trxFilePath)} (<a href=\"{url}\">Logs</a>)";
+
+        reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"### {title}");
         reportBuilder.AppendLine("| Passed | Failed | Skipped | Total |");
         reportBuilder.AppendLine("|--------|--------|---------|-------|");
         reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"| {passed} | {failed} | {skipped} | {total} |");
 
         reportBuilder.AppendLine();
         if (testRun.Results?.UnitTestResults is null)
         {
+            Console.WriteLine($"Could not find any UnitTestResult entries in {trxFilePath}");
             return;
         }
 
@@ -132,18 +138,20 @@ public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuild
         {
             foreach (var test in failedTests)
             {
-                var title = string.IsNullOrEmpty(url)
-                                ? $"🔴 <b>{test.TestName}</b>"
-                                : $"🔴 <a href=\"{url}\">{test.TestName}</a>";
-
                 reportBuilder.AppendLine("<div>");
                 reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"""
-                    <details><summary>{title}</summary>
+                    <details><summary>🔴 <b>{test.TestName}</b></summary>
+
                 """);
 
                 var errorMsgBuilder = new StringBuilder();
                 errorMsgBuilder.AppendLine(test.Output?.ErrorInfo?.InnerText ?? string.Empty);
-                errorMsgBuilder.AppendLine(test.Output?.StdOut ?? string.Empty);
+                if (test.Output?.StdOut is not null)
+                {
+                    errorMsgBuilder.AppendLine();
+                    errorMsgBuilder.AppendLine("### StdOut");
+                    errorMsgBuilder.AppendLine(test.Output.StdOut);
+                }
 
                 // Truncate long error messages for readability
                 var errorMsgTruncated = TruncateTheStart(errorMsgBuilder.ToString(), 50_000);