From 46869675952c533d28832462fa7945e623802de8 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Mon, 19 Dec 2022 15:11:52 -0800 Subject: [PATCH 01/21] Temp log gets written as tests run, rather than just at the end. --- .../Common/XUnitWrapperGenerator/ITestInfo.cs | 6 +- .../XUnitWrapperGenerator.cs | 12 +- .../Common/XUnitWrapperLibrary/TestSummary.cs | 136 ++++++++++++------ 3 files changed, 110 insertions(+), 44 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 213fc3d36d1bb..6654d597ff4b9 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -341,11 +341,11 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();"); builder.AppendLine(testExecutionExpression); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());"); + builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw);"); builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); builder.AppendLine("catch (System.Exception ex) {"); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());"); + builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw);"); builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); @@ -359,6 +359,6 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe public string GenerateSkippedTestReporting(ITestInfo skippedTest) { - return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\", System.TimeSpan.Zero, string.Empty);"; + return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\", System.TimeSpan.Zero, string.Empty, tempLogSw);"; } } diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index a76025db7d727..055d569ca5689 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -160,6 +160,9 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}_templog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}_templog.xml"");"); + ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); StringBuilder testExecutorBuilder = new(); @@ -177,16 +180,24 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos if (testsLeftInCurrentTestExecutor == 0) { if (currentTestExecutor != 0) + { + testExecutorBuilder.AppendLine("}"); testExecutorBuilder.AppendLine("}"); + } + currentTestExecutor++; testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(){{"); + testExecutorBuilder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}_templog.xml"")) {{"); builder.AppendLine($"TestExecutor{currentTestExecutor}();"); testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empircally seems to work well } + testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter)); totalTestsEmitted++; testsLeftInCurrentTestExecutor--; } + + testExecutorBuilder.AppendLine("}"); testExecutorBuilder.AppendLine("}"); } @@ -197,7 +208,6 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine("return 100;"); builder.Append(testExecutorBuilder); - builder.AppendLine("public static class TestCount { public const int Count = " + totalTestsEmitted.ToString() + "; }"); return builder.ToString(); } diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index b230fa485a055..6f429f0d02e66 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -3,40 +3,127 @@ // using System; +using System.IO; using System.Collections.Generic; using System.Text; namespace XUnitWrapperLibrary; public class TestSummary { - readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason, string? Output); + // readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason, string? Output); + readonly record struct TestResult + { + readonly string Name; + readonly string ContainingTypeName; + readonly string MethodName; + readonly TimeSpan Duration; + readonly Exception? Exception; + readonly string? SkipReason; + readonly string? Output; + + public TestResult(string name, string containingTypeName, string methodName, TimeSpan duration, Exception? exception, string? skipReason, string? output) + { + Name = name; + ContainingTypeName = containingTypeName; + MethodName = methodName; + Duration = duration; + Exception = exception; + SkipReason = skipReason; + Output = output; + } + + public string ToXmlString() + { + var testResultSb = new StringBuilder(); + testResultSb.Append($@"" + : string.Empty; + + if (Exception is not null) + { + string? message = Exception.Message; + + if (Exception is System.Reflection.TargetInvocationException tie) + { + if (tie.InnerException is not null) + { + message = $"{message}\n INNER EXCEPTION--\n" + + $"{tie.InnerException.GetType()}--\n" + + $"{tie.InnerException.Message}--\n" + + $"{tie.InnerException.StackTrace}"; + } + } + + if (string.IsNullOrWhiteSpace(message)) + { + message = "NoExceptionMessage"; + } + + testResultSb.Append($@" result=""Fail"">" + + $@"" + + $"" + + "{outputElement}"); + } + else if (SkipReason is not null) + { + testResultSb.Append($@" result=""Skip"">"); + } + else + { + testResultSb.AppendLine($@" result=""Pass"">{outputElement}"); + } + + return testResultSb.ToString(); + } + } public int PassedTests { get; private set; } = 0; public int FailedTests { get; private set; } = 0; public int SkippedTests { get; private set; } = 0; private readonly List _testResults = new(); - private DateTime _testRunStart = DateTime.Now; - public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string output) + public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string output, StreamWriter tempLogSw) { PassedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, null, output)); + var result = new TestResult(name, containingTypeName, methodName, duration, null, null, output); + _testResults.Add(result); + tempLogSw.WriteLine(result.ToXmlString()); } - public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex, string output) + public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex, string output, StreamWriter tempLogSw) { FailedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, ex, null, output)); + var result = new TestResult(name, containingTypeName, methodName, duration, ex, null, output); + _testResults.Add(result); + tempLogSw.WriteLine(result.ToXmlString()); } - public void ReportSkippedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string reason) + public void ReportSkippedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string reason, StreamWriter tempLogSw) { SkippedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, reason, null)); + var result = new TestResult(name, containingTypeName, methodName, duration, null, reason, null); + _testResults.Add(result); + tempLogSw.WriteLine(result.ToXmlString()); } + // NOTE: This will likely change or be removed altogether with the existence of the temp log. public string GetTestResultOutput(string assemblyName) { double totalRunSeconds = (DateTime.Now - _testRunStart).TotalSeconds; @@ -69,38 +156,7 @@ public string GetTestResultOutput(string assemblyName) foreach (var test in _testResults) { - resultsFile.Append($@"" : string.Empty; - if (test.Exception is not null) - { - string exceptionMessage = test.Exception.Message; - if (test.Exception is System.Reflection.TargetInvocationException tie) - { - if (tie.InnerException != null) - { - exceptionMessage = $"{exceptionMessage} \n INNER EXCEPTION--\n {tie.InnerException.GetType()}--\n{tie.InnerException.Message}--\n{tie.InnerException.StackTrace}"; - } - } - if (string.IsNullOrWhiteSpace(exceptionMessage)) - { - exceptionMessage = "NoExceptionMessage"; - } - - string? stackTrace = test.Exception.StackTrace; - if (string.IsNullOrWhiteSpace(stackTrace)) - { - stackTrace = "NoStackTrace"; - } - resultsFile.AppendLine($@"result=""Fail"">{outputElement}"); - } - else if (test.SkipReason is not null) - { - resultsFile.AppendLine($@"result=""Skip"">"); - } - else - { - resultsFile.AppendLine($@" result=""Pass"">{outputElement}"); - } + resultsFile.AppendLine(test.ToXmlString()); } resultsFile.AppendLine(""); From 10747d938f8e3a08b2cfde7c3b192d12f002ba33 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Fri, 6 Jan 2023 17:01:55 -0800 Subject: [PATCH 02/21] Added placeholder for the XUnit log checker and it now builds with src/tests/build.sh --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 47 +++++++++++++++++++ .../XUnitLogChecker/XUnitLogChecker.csproj | 12 +++++ src/tests/build.proj | 5 ++ 3 files changed, 64 insertions(+) create mode 100644 src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs create mode 100644 src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs new file mode 100644 index 0000000000000..13d07b1c63870 --- /dev/null +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using System.Xml; + +public class XUnitLogChecker +{ + private static class Patterns + { + public const string OpenTag = @"(\B<\w+)|(\B)|(\]\]>)"; + } + + const int SUCCESS = 0; + const int FAILURE = -1; + + static int Main(string[] args) + { + if (TryLoadXml(args[0])) + { + Console.WriteLine($"The given XML '{args[0]}' is well formed! Exiting..."); + return SUCCESS; + } + + return FAILURE; + } + + static bool TryLoadXml(string xFile) + { + try + { + using (var xReader = XmlReader.Create(new StreamReader(xFile))) + { + var xDocument = new XmlDocument(); + xDocument.Load(xReader); + } + } + + catch (Exception ex) + { + Console.WriteLine($"The given XML {xFile} was malformed. Fixing now..."); + return false; + } + + return true; + } +} + diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj new file mode 100644 index 0000000000000..b55b68941a8fd --- /dev/null +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj @@ -0,0 +1,12 @@ + + + + Exe + $(NetCoreAppToolCurrent) + + + + + + + diff --git a/src/tests/build.proj b/src/tests/build.proj index 63b799dc835d0..b37b17bbd0fd9 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -25,6 +25,7 @@ + @@ -509,6 +510,10 @@ Projects="$(MSBuildProjectFile)" Targets="EmitTestExclusionList" Properties="XunitTestBinBase=$(XunitTestBinBase)" /> + + Date: Fri, 6 Jan 2023 17:53:29 -0800 Subject: [PATCH 03/21] XUnit Log Checker is now functional in the repo. Next step is to bundle it and have Helix use it as needed. --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 77 +++++++++++++++++-- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index 13d07b1c63870..649d2d375dd55 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text.RegularExpressions; using System.Xml; public class XUnitLogChecker @@ -10,18 +13,24 @@ private static class Patterns public const string CloseTag = @"(\B)|(\]\]>)"; } - const int SUCCESS = 0; - const int FAILURE = -1; - static int Main(string[] args) { + if (args.Count() < 1) + { + Console.WriteLine("The path to the XML log file to be checked is required."); + return -1; + } + + // Check if the XUnit log is well-formed. If yes, then we have nothing + // else to do :) if (TryLoadXml(args[0])) { Console.WriteLine($"The given XML '{args[0]}' is well formed! Exiting..."); - return SUCCESS; + return 0; } - return FAILURE; + FixTheXml(args[0]); + return 0; } static bool TryLoadXml(string xFile) @@ -34,14 +43,68 @@ static bool TryLoadXml(string xFile) xDocument.Load(xReader); } } - catch (Exception ex) { Console.WriteLine($"The given XML {xFile} was malformed. Fixing now..."); return false; } - return true; } + + static void FixTheXml(string xFile) + { + var tags = new Stack(); + + foreach (string line in File.ReadLines(xFile)) + { + // Get all XML tags found in the current line. + var opens = Regex.Matches(line, Patterns.OpenTag).ToList(); + var closes = Regex.Matches(line, Patterns.CloseTag).ToList(); + + foreach (Match m in opens) + { + // Push the next opening tag to the stack. We need only the actual + // tag without the opening and closing symbols, so we ask LINQ to + // lend us a hand. + tags.Push(new String(m.Value.Where(c => char.IsLetter(c)).ToArray())); + } + + // If we found any closing tags, then ensure they match their respective + // opening ones before continuing the analysis. + while (closes.Count > 0) + { + string nextClosing = closes.First().Value.Equals("]]>") + ? "CDATA" + : new String(closes.First() + .Value + .Where(c => char.IsLetter(c)) + .ToArray()); + + if (nextClosing.Equals(tags.Peek())) + { + tags.Pop(); + closes.RemoveAt(0); + } + } + } + + if (tags.Count == 0) + { + Console.WriteLine($"\nXUnit log file '{xFile}' was A-OK!\n"); + } + + // Write the closings for all the lone opened tags we found. + using (StreamWriter xsw = File.AppendText(xFile)) + while (tags.Count > 0) + { + string tag = tags.Pop(); + if (tag.Equals("CDATA")) + xsw.WriteLine("]]>"); + else + xsw.WriteLine($""); + } + + Console.WriteLine("\nXUnit log file has been fixed!\n"); + } } From b628a21789bde33e3bcbbd5e0272401388906ff4 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 10 Jan 2023 17:13:47 -0800 Subject: [PATCH 04/21] Making progress in packaging to Helix. --- src/tests/Common/helixpublishwitharcade.proj | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 8a0edff833487..cb6e2fb3c9df1 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -65,6 +65,9 @@ $(TestBinDir)Tests\Core_Root\ + $([MSBuild]::NormalizeDirectory($(CoreRootDirectory))) + $(TestBinDir)Common\XUnitLogChecker\XUnitLogChecker\ + $([MSBuild]::NormalizeDirectory($(XUnitLogCheckerDirectory))) $(TestBinDir)LegacyPayloads\ $([MSBuild]::NormalizeDirectory($(LegacyPayloadsRootDirectory))) $(TestBinDir)MergedPayloads\ @@ -358,6 +361,7 @@ <_MergedWrapperRunScript Include="$([System.IO.Path]::ChangeExtension('%(_MergedWrapperMarker.Identity)', '.$(TestScriptExtension)'))" /> + <_MergedWrapperDirectory>$([System.IO.Path]::GetDirectoryName('%(_MergedWrapperRunScript.Identity)')) <_MergedWrapperParentDirectory>$([System.IO.Path]::GetDirectoryName('$(_MergedWrapperDirectory)')) @@ -366,6 +370,7 @@ <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'">call + <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" /> <_MergedWrapperOutOfProcessTestFiles @@ -390,6 +395,11 @@ + + XUnitLogChecker + $(XUnitLogCheckerExe).exe + + @@ -398,10 +408,26 @@ + + + + @@ -698,6 +724,7 @@ + From 8bbacb64457ceae6bff2a55c09f88bf62ccee43f Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Wed, 11 Jan 2023 17:55:17 -0800 Subject: [PATCH 05/21] Began adapting the log checker to the actual Helix context. --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 36 ++++++++++++++----- src/tests/Common/helixpublishwitharcade.proj | 13 +++---- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index 649d2d375dd55..2f0714e0af584 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -15,21 +15,40 @@ private static class Patterns static int Main(string[] args) { - if (args.Count() < 1) + if (args.Count() < 2) { - Console.WriteLine("The path to the XML log file to be checked is required."); + Console.WriteLine("The path to the log file and the name of the wrapper" + + " are required for an accurate check and fixing."); return -1; } + string resultsDir = args[0]; + string wrapperName = args[1]; + + if (!Directory.Exists(resultsDir)) + { + Console.WriteLine($"The given path '{resultsDir}' was not found."); + return -2; + } + // Check if the XUnit log is well-formed. If yes, then we have nothing // else to do :) - if (TryLoadXml(args[0])) - { - Console.WriteLine($"The given XML '{args[0]}' is well formed! Exiting..."); - return 0; - } + // if (TryLoadXml(args[0])) + // { + // Console.WriteLine($"The given XML '{args[0]}' is well formed! Exiting..."); + // return 0; + // } + + string tempLogName = $"{wrapperName}_templog.xml"; + string finalLogName = $"{wrapperName}.testResults.xml"; + + if (File.Exists(Path.Combine(resultsDir, finalLogName))) + { + Console.WriteLine($"Item '{wrapperName}' did complete successfully!"); + return 0; + } - FixTheXml(args[0]); + FixTheXml(Path.Combine(resultsDir, tempLogName)); return 0; } @@ -51,6 +70,7 @@ static bool TryLoadXml(string xFile) return true; } + // Missing Stuff: Write the final log file, not append to the temporary one. static void FixTheXml(string xFile) { var tags = new Stack(); diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index cb6e2fb3c9df1..37a3d94415d4b 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -395,11 +395,6 @@ - - XUnitLogChecker - $(XUnitLogCheckerExe).exe - - @@ -408,10 +403,10 @@ - + - + - + From 786ed02d79213efc8e26b2f6a0e5620b24ac36b4 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Fri, 13 Jan 2023 17:11:14 -0800 Subject: [PATCH 06/21] XML Fixer builds and works nowgit add *! --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 66 +++++++------------ .../XUnitLogChecker/XUnitLogChecker.csproj | 1 + .../XUnitWrapperGenerator.cs | 6 +- 3 files changed, 28 insertions(+), 45 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index 2f0714e0af584..0326d48508661 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -18,56 +18,38 @@ static int Main(string[] args) if (args.Count() < 2) { Console.WriteLine("The path to the log file and the name of the wrapper" - + " are required for an accurate check and fixing."); + + " are required for an accurate check and fixing."); return -1; } - string resultsDir = args[0]; - string wrapperName = args[1]; - - if (!Directory.Exists(resultsDir)) - { - Console.WriteLine($"The given path '{resultsDir}' was not found."); - return -2; - } - - // Check if the XUnit log is well-formed. If yes, then we have nothing - // else to do :) - // if (TryLoadXml(args[0])) - // { - // Console.WriteLine($"The given XML '{args[0]}' is well formed! Exiting..."); - // return 0; - // } - - string tempLogName = $"{wrapperName}_templog.xml"; - string finalLogName = $"{wrapperName}.testResults.xml"; - - if (File.Exists(Path.Combine(resultsDir, finalLogName))) - { - Console.WriteLine($"Item '{wrapperName}' did complete successfully!"); - return 0; - } - - FixTheXml(Path.Combine(resultsDir, tempLogName)); - return 0; - } + string resultsDir = args[0]; + string wrapperName = args[1]; - static bool TryLoadXml(string xFile) - { - try + string tempLogName = $"{wrapperName}.templog.xml"; + string finalLogName = $"{wrapperName}.testResults.xml"; + + string tempLogPath = Path.Combine(resultsDir, tempLogName); + string finalLogPath = Path.Combine(resultsDir, finalLogName); + + if (File.Exists(finalLogPath)) { - using (var xReader = XmlReader.Create(new StreamReader(xFile))) - { - var xDocument = new XmlDocument(); - xDocument.Load(xReader); - } + Console.WriteLine($"Item '{wrapperName}' did complete successfully!"); + return 0; } - catch (Exception ex) + + if (!File.Exists(tempLogPath)) { - Console.WriteLine($"The given XML {xFile} was malformed. Fixing now..."); - return false; + Console.WriteLine("No logs were found. Something went very wrong" + + " with this item..."); + return -2; } - return true; + + FixTheXml(tempLogPath); + + // Rename the temp log to the final log, so that Helix can use it without + // knowing what transpired here. + File.Move(tempLogPath, finalLogPath); + return 0; } // Missing Stuff: Write the final log file, not append to the temporary one. diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj index b55b68941a8fd..d339f492d63d3 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.csproj @@ -3,6 +3,7 @@ Exe $(NetCoreAppToolCurrent) + true diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 055d569ca5689..772a789d2a6ab 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -160,8 +160,8 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); - builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}_templog.xml""))"); - builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}_templog.xml"");"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.templog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.templog.xml"");"); ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); @@ -187,7 +187,7 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos currentTestExecutor++; testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(){{"); - testExecutorBuilder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}_templog.xml"")) {{"); + testExecutorBuilder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml"")) {{"); builder.AppendLine($"TestExecutor{currentTestExecutor}();"); testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empircally seems to work well } From e2e2a272909e95d3b953b9cc0198e9892150efea Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 17 Jan 2023 11:47:31 -0800 Subject: [PATCH 07/21] Forgot to replace the template with the actual values in helixpublishwitharcade.proj --- src/tests/Common/helixpublishwitharcade.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 37a3d94415d4b..a8b9e76e41ec7 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -403,7 +403,8 @@ - + + From 1b991fcbd120cf66533d7cdb4d0f77eef5b65a61 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Thu, 19 Jan 2023 11:12:05 -0800 Subject: [PATCH 08/21] Fixed a bug with unclosed quotes in the log checker --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index 0326d48508661..b8b770fb41844 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -52,7 +52,6 @@ static int Main(string[] args) return 0; } - // Missing Stuff: Write the final log file, not append to the temporary one. static void FixTheXml(string xFile) { var tags = new Stack(); @@ -68,7 +67,33 @@ static void FixTheXml(string xFile) // Push the next opening tag to the stack. We need only the actual // tag without the opening and closing symbols, so we ask LINQ to // lend us a hand. - tags.Push(new String(m.Value.Where(c => char.IsLetter(c)).ToArray())); + string nextOpen = new String(m.Value + .Where(c => char.IsLetter(c)) + .ToArray()); + + // During testing, I encountered a case where one of the labels + // a 'test' tag, had incomplete quotes. Since this can only happen + // with 'test' tags, we handle this case separately. It's also + // worth noting that in theory, this case can only happen if it's + // the last line in the log. But to be safe, we'll check all the + // 'test' tags. + if (nextOpen.Equals("test")) + { + int quoteCount = 0; + + // Good ol' simple loop showed better performance in counting + // the quotes. + foreach (char c in line.AsSpan()) + { + if (c == '"') + quoteCount++; + } + + if ((quoteCount % 2) != 0) + nextOpen = "testWithMissingQuote"; + } + + tags.Push(nextOpen); } // If we found any closing tags, then ensure they match their respective @@ -102,6 +127,8 @@ static void FixTheXml(string xFile) string tag = tags.Pop(); if (tag.Equals("CDATA")) xsw.WriteLine("]]>"); + else if (tag.Equals("testWithMissingQuote")) + xsw.WriteLine("\""); else xsw.WriteLine($""); } From 01d9dcf60a907c7dad8f87b06e8ebc2a349270ac Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Mon, 23 Jan 2023 14:31:22 -0800 Subject: [PATCH 09/21] Fixed wrong log paths in Helix results. --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 18 ++++++++++-------- src/tests/Common/helixpublishwitharcade.proj | 13 +++++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index b8b770fb41844..b54318b870928 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -17,8 +17,9 @@ static int Main(string[] args) { if (args.Count() < 2) { - Console.WriteLine("The path to the log file and the name of the wrapper" - + " are required for an accurate check and fixing."); + Console.WriteLine("[XUnitLogChecker]: The path to the log file and" + + " the name of the wrapper are required for an" + + " accurate check and fixing."); return -1; } @@ -30,17 +31,18 @@ static int Main(string[] args) string tempLogPath = Path.Combine(resultsDir, tempLogName); string finalLogPath = Path.Combine(resultsDir, finalLogName); - + if (File.Exists(finalLogPath)) { - Console.WriteLine($"Item '{wrapperName}' did complete successfully!"); + Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did" + + " complete successfully!"); return 0; } if (!File.Exists(tempLogPath)) { - Console.WriteLine("No logs were found. Something went very wrong" - + " with this item..."); + Console.WriteLine("[XUnitLogChecker]: No logs were found. Something" + + " went very wrong with this item..."); return -2; } @@ -117,7 +119,7 @@ static void FixTheXml(string xFile) if (tags.Count == 0) { - Console.WriteLine($"\nXUnit log file '{xFile}' was A-OK!\n"); + Console.WriteLine($"[XUnitLogChecker]: XUnit log file '{xFile}' was A-OK!"); } // Write the closings for all the lone opened tags we found. @@ -133,7 +135,7 @@ static void FixTheXml(string xFile) xsw.WriteLine($""); } - Console.WriteLine("\nXUnit log file has been fixed!\n"); + Console.WriteLine("[XUnitLogChecker]: XUnit log file has been fixed!"); } } diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index a8b9e76e41ec7..2dc018b39664d 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -366,7 +366,10 @@ <_MergedWrapperDirectory>$([System.IO.Path]::GetDirectoryName('%(_MergedWrapperRunScript.Identity)')) <_MergedWrapperParentDirectory>$([System.IO.Path]::GetDirectoryName('$(_MergedWrapperDirectory)')) <_MergedWrapperName>%(_MergedWrapperRunScript.FileName) + <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath))) + <_MergedWrapperRunScriptDirectoryRelative Condition="'$(_MergedWrapperRunScriptRelative)' != ''">$([System.IO.Path]::GetDirectoryName($(_MergedWrapperRunScriptRelative))) + <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'">call @@ -395,6 +398,12 @@ + + dotnet %24HELIX_CORRELATION_PAYLOAD/ + dotnet %25HELIX_CORRELATION_PAYLOAD%25/ + $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName) + + @@ -403,10 +412,10 @@ - - + + From b550c77772c83bfdf61aa580c117bc995237cb45 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 24 Jan 2023 10:12:07 -0800 Subject: [PATCH 10/21] Added missing Wasm logger stuff. --- src/tests/Common/helixpublishwitharcade.proj | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 2dc018b39664d..f59923e4e00c6 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -479,8 +479,10 @@ <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) <_MergedWrapperName>%(_MergedWrapperMarker.FileName) + <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)AppBundle', %(_MergedWrapperRunScript.FullPath))) <_MergedWrapperRunScriptRelative Condition="'$(TestWrapperTargetsWindows)' != 'true'">./$(_MergedWrapperRunScriptRelative) + <_MergedWrapperRunScriptDirectoryRelative Condition="'$(_MergedWrapperRunScriptRelative)' != ''">$([System.IO.Path]::GetDirectoryName($(_MergedWrapperRunScriptRelative))) @@ -496,11 +498,18 @@ <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> + + dotnet %24HELIX_CORRELATION_PAYLOAD/ + dotnet %25HELIX_CORRELATION_PAYLOAD%25/ + $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName) + + + From 7c2753e89a91bd3bdd69335cdb179744c5dee57b Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 24 Jan 2023 14:18:17 -0800 Subject: [PATCH 11/21] Fixed a mismatching tag in the log generator, that occurred when recording skipped tests. --- src/tests/Common/XUnitWrapperLibrary/TestSummary.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index 6f429f0d02e66..5ec4997b12b90 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -81,7 +81,7 @@ public string ToXmlString() ? SkipReason : "No Known Skip Reason"); - testResultSb.AppendLine("]]"); + testResultSb.AppendLine("]]>"); } else { From 9944bdb1ae3bca73dc11fb55ecdf3100935679a0 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 24 Jan 2023 17:23:23 -0800 Subject: [PATCH 12/21] Removed some messaging for dev-logging purposes, and fixed a problem where Mono was being used instead of CoreCLR. --- src/tests/Common/helixpublishwitharcade.proj | 26 ++++---------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index e9c19fd4f8b35..bb7f6134aa17c 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -66,7 +66,7 @@ $(TestBinDir)Tests\Core_Root\ $([MSBuild]::NormalizeDirectory($(CoreRootDirectory))) - $(TestBinDir)Common\XUnitLogChecker\XUnitLogChecker\ + $(TestBinDir)Common\XUnitLogChecker\ $([MSBuild]::NormalizeDirectory($(XUnitLogCheckerDirectory))) $(TestBinDir)LegacyPayloads\ $([MSBuild]::NormalizeDirectory($(LegacyPayloadsRootDirectory))) @@ -396,8 +396,8 @@ - dotnet %24HELIX_CORRELATION_PAYLOAD/ - dotnet %25HELIX_CORRELATION_PAYLOAD%25/ + dotnet %24HELIX_CORRELATION_PAYLOAD/XUnitLogChecker/ + dotnet %25HELIX_CORRELATION_PAYLOAD%25/XUnitLogChecker/ $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName) @@ -412,24 +412,8 @@ - - - - - - - - - - - - - - - - @@ -496,8 +480,8 @@ - dotnet %24HELIX_CORRELATION_PAYLOAD/ - dotnet %25HELIX_CORRELATION_PAYLOAD%25/ + dotnet %24HELIX_CORRELATION_PAYLOAD/XUnitLogChecker/ + dotnet %25HELIX_CORRELATION_PAYLOAD%25/XUnitLogChecker/ $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName) From f344c3151046509dc8b1596ae079c488360355f2 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Wed, 25 Jan 2023 17:27:11 -0800 Subject: [PATCH 13/21] Added missing stream writer for the temp log in the XHarnessRunner. --- .../XUnitWrapperGenerator/XUnitWrapperGenerator.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 772a789d2a6ab..f7f3791d9d86c 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -230,6 +230,9 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.templog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.templog.xml"");"); + ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); StringBuilder testExecutorBuilder = new(); @@ -238,6 +241,9 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI if (testInfos.Length > 0) { + // Open the stream writer for the temp log. + builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml"")) {{"); + // Break tests into groups of 50 so that we don't create an unreasonably large main method // Excessively large methods are known to take a long time to compile, and use excessive stack // leading to test failures. @@ -248,14 +254,15 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI if (currentTestExecutor != 0) testExecutorBuilder.AppendLine("}"); currentTestExecutor++; - testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(XUnitWrapperLibrary.TestSummary summary, XUnitWrapperLibrary.TestFilter filter, XUnitWrapperLibrary.TestOutputRecorder outputRecorder, System.Diagnostics.Stopwatch stopwatch){{"); - builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch);"); + testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(XUnitWrapperLibrary.TestSummary summary, XUnitWrapperLibrary.TestFilter filter, XUnitWrapperLibrary.TestOutputRecorder outputRecorder, System.Diagnostics.Stopwatch stopwatch, System.IO.StreamWriter tempLogSw){{"); + builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch, tempLogSw);"); testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well } testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter)); testsLeftInCurrentTestExecutor--; } testExecutorBuilder.AppendLine("}"); + builder.AppendLine("}"); } builder.AppendLine("return summary;"); From 13a9cb7e3a03f0782d23a54e94e5423303823052 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Mon, 30 Jan 2023 16:44:50 -0800 Subject: [PATCH 14/21] Added recorder of test counters. --- .../Common/XUnitWrapperGenerator/ITestInfo.cs | 16 +++++-- .../XUnitWrapperGenerator.cs | 23 +++++---- .../Common/XUnitWrapperLibrary/TestSummary.cs | 47 +++++++++++++++---- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 6654d597ff4b9..81e07deb6041e 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -333,7 +333,8 @@ public WrapperLibraryTestSummaryReporting(string summaryLocalIdentifier, string public string WrapTestExecutionWithReporting(string testExecutionExpression, ITestInfo test) { StringBuilder builder = new(); - builder.AppendLine($"if ({_filterLocalIdentifier} is null || {_filterLocalIdentifier}.ShouldRunTest(@\"{test.ContainingType}.{test.Method}\", {test.TestNameExpression}))"); + builder.AppendLine($"if ({_filterLocalIdentifier} is null || {_filterLocalIdentifier}.ShouldRunTest(@\"{test.ContainingType}.{test.Method}\"," + + $" {test.TestNameExpression}))"); builder.AppendLine("{"); builder.AppendLine($"System.TimeSpan testStart = stopwatch.Elapsed;"); @@ -341,11 +342,17 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();"); builder.AppendLine(testExecutionExpression); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw);"); + + builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\"," + + $" stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); builder.AppendLine("catch (System.Exception ex) {"); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw);"); + + builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\"," + + $" stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);"); + builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});"); builder.AppendLine("}"); @@ -359,6 +366,7 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe public string GenerateSkippedTestReporting(ITestInfo skippedTest) { - return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\", System.TimeSpan.Zero, string.Empty, tempLogSw);"; + return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\"," + + $" System.TimeSpan.Zero, string.Empty, tempLogSw, statsCsvSw);"; } } diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index f7f3791d9d86c..1ae6164717e07 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -154,14 +154,21 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};"))); builder.AppendLine("System.Collections.Generic.HashSet testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.yml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.tempLog.yml"");"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.testStats.csv""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.testStats.csv"");"); + builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args, testExclusionList);"); + // builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new(TestCount.Count);"); builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();"); builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();"); builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); - builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.templog.xml""))"); - builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.templog.xml"");"); + builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.tempLog.yml""))"); + builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{"); + builder.AppendLine("statsCsvSw.WriteLine(TestCount.Count,0,0,0);"); ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); @@ -182,14 +189,12 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos if (currentTestExecutor != 0) { testExecutorBuilder.AppendLine("}"); - testExecutorBuilder.AppendLine("}"); } currentTestExecutor++; - testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(){{"); - testExecutorBuilder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml"")) {{"); - builder.AppendLine($"TestExecutor{currentTestExecutor}();"); - testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empircally seems to work well + testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(System.IO.StreamWriter tempLogSw, System.IO.StreamWriter statsCsvSw) {{"); + builder.AppendLine($"TestExecutor{currentTestExecutor}(tempLogSw, statsCsvSw);"); + testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well } testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter)); @@ -198,9 +203,11 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos } testExecutorBuilder.AppendLine("}"); - testExecutorBuilder.AppendLine("}"); } + // Closing the 'using' statements that stream the temporary files. + builder.AppendLine("}"); + builder.AppendLine($@"string testResults = summary.GetTestResultOutput(""{assemblyName}"");"); builder.AppendLine($@"string workitemUploadRoot = System.Environment.GetEnvironmentVariable(""HELIX_WORKITEM_UPLOAD_ROOT"");"); builder.AppendLine($@"if (workitemUploadRoot != null) System.IO.File.WriteAllText(System.IO.Path.Combine(workitemUploadRoot, ""{assemblyName}.testResults.xml.txt""), testResults);"); diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index 5ec4997b12b90..f0ca25e9a5d03 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -10,7 +10,6 @@ namespace XUnitWrapperLibrary; public class TestSummary { - // readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason, string? Output); readonly record struct TestResult { readonly string Name; @@ -21,7 +20,13 @@ readonly record struct TestResult readonly string? SkipReason; readonly string? Output; - public TestResult(string name, string containingTypeName, string methodName, TimeSpan duration, Exception? exception, string? skipReason, string? output) + public TestResult(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + Exception? exception, + string? skipReason, + string? output) { Name = name; ContainingTypeName = containingTypeName; @@ -95,32 +100,58 @@ public string ToXmlString() public int PassedTests { get; private set; } = 0; public int FailedTests { get; private set; } = 0; public int SkippedTests { get; private set; } = 0; + public int TotalTests { get; private set; } = 0; private readonly List _testResults = new(); private DateTime _testRunStart = DateTime.Now; - public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string output, StreamWriter tempLogSw) + public void ReportPassedTest(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + string output, + StreamWriter tempLogSw, + StreamWriter statsCsvSw) { PassedTests++; + TotalTests++; var result = new TestResult(name, containingTypeName, methodName, duration, null, null, output); _testResults.Add(result); - tempLogSw.WriteLine(result.ToXmlString()); + // tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.WriteLine(TotalTests, PassedTests, FailedTests, SkippedTests); } - public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex, string output, StreamWriter tempLogSw) + public void ReportFailedTest(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + Exception ex, + string output, + StreamWriter tempLogSw, + StreamWriter statsCsvSw) { FailedTests++; + TotalTests++; var result = new TestResult(name, containingTypeName, methodName, duration, ex, null, output); _testResults.Add(result); - tempLogSw.WriteLine(result.ToXmlString()); + // tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.WriteLine(TotalTests, PassedTests, FailedTests, SkippedTests); } - public void ReportSkippedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string reason, StreamWriter tempLogSw) + public void ReportSkippedTest(string name, + string containingTypeName, + string methodName, + TimeSpan duration, + string reason, + StreamWriter tempLogSw, + StreamWriter statsCsvSw) { SkippedTests++; + TotalTests++; var result = new TestResult(name, containingTypeName, methodName, duration, null, reason, null); _testResults.Add(result); - tempLogSw.WriteLine(result.ToXmlString()); + // tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.WriteLine(TotalTests, PassedTests, FailedTests, SkippedTests); } // NOTE: This will likely change or be removed altogether with the existence of the temp log. From 56febb34c8301b5ddbf17ac260bf66db3f053d6b Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 31 Jan 2023 16:37:27 -0800 Subject: [PATCH 15/21] Xml's and Csv's work fine now --- .../XUnitWrapperGenerator.cs | 13 +++++++----- .../Common/XUnitWrapperLibrary/TestSummary.cs | 21 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 1ae6164717e07..14c38d092056c 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -154,10 +154,12 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};"))); builder.AppendLine("System.Collections.Generic.HashSet testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();"); - builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.yml""))"); - builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.tempLog.yml"");"); + builder.Append("\n"); // Make the FullRunner.g.cs file a bit more readable. + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.tempLog.xml"");"); builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.testStats.csv""))"); builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.testStats.csv"");"); + builder.Append("\n"); builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args, testExclusionList);"); // builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new(TestCount.Count);"); @@ -166,9 +168,10 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); - builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.tempLog.yml""))"); + builder.Append("\n"); + builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.tempLog.xml""))"); builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{"); - builder.AppendLine("statsCsvSw.WriteLine(TestCount.Count,0,0,0);"); + builder.AppendLine("statsCsvSw.WriteLine($\"{TestCount.Count},0,0,0\");"); ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); @@ -206,7 +209,7 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos } // Closing the 'using' statements that stream the temporary files. - builder.AppendLine("}"); + builder.AppendLine("}\n"); builder.AppendLine($@"string testResults = summary.GetTestResultOutput(""{assemblyName}"");"); builder.AppendLine($@"string workitemUploadRoot = System.Environment.GetEnvironmentVariable(""HELIX_WORKITEM_UPLOAD_ROOT"");"); diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index f0ca25e9a5d03..c81a82d99423d 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -117,8 +117,11 @@ public void ReportPassedTest(string name, TotalTests++; var result = new TestResult(name, containingTypeName, methodName, duration, null, null, output); _testResults.Add(result); - // tempLogSw.WriteLine(result.ToXmlString()); - statsCsvSw.WriteLine(TotalTests, PassedTests, FailedTests, SkippedTests); + + statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}"); + tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.Flush(); + tempLogSw.Flush(); } public void ReportFailedTest(string name, @@ -134,8 +137,11 @@ public void ReportFailedTest(string name, TotalTests++; var result = new TestResult(name, containingTypeName, methodName, duration, ex, null, output); _testResults.Add(result); - // tempLogSw.WriteLine(result.ToXmlString()); - statsCsvSw.WriteLine(TotalTests, PassedTests, FailedTests, SkippedTests); + + statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}"); + tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.Flush(); + tempLogSw.Flush(); } public void ReportSkippedTest(string name, @@ -150,8 +156,11 @@ public void ReportSkippedTest(string name, TotalTests++; var result = new TestResult(name, containingTypeName, methodName, duration, null, reason, null); _testResults.Add(result); - // tempLogSw.WriteLine(result.ToXmlString()); - statsCsvSw.WriteLine(TotalTests, PassedTests, FailedTests, SkippedTests); + + statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}"); + tempLogSw.WriteLine(result.ToXmlString()); + statsCsvSw.Flush(); + tempLogSw.Flush(); } // NOTE: This will likely change or be removed altogether with the existence of the temp log. From 45e2bd0819ebc5ffdfeb97aa342a2bb11cf599df Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Tue, 7 Feb 2023 18:03:42 -0800 Subject: [PATCH 16/21] Began new approach... --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 228 ++++++++++++------ 1 file changed, 154 insertions(+), 74 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index b54318b870928..7ec60cdf82903 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -9,133 +9,213 @@ public class XUnitLogChecker { private static class Patterns { - public const string OpenTag = @"(\B<\w+)|(\B)|(\]\]>)"; } + private readonly struct TagResult + { + public TagResult(string matchValue, TagCategory matchCategory) + { + Value = matchValue; + Category = matchCategory; + } + + public string Value { get; init; } + public TagCategory Category { get; init; } + } + + private enum TagCategory { OPENING, CLOSING } + + private const int SUCCESS = 0; + private const int MISSING_ARGS = -1; + private const int SOMETHING_VERY_WRONG = -2; + static int Main(string[] args) { - if (args.Count() < 2) + // Maybe add a '--help' flag that also gets triggered in this case. + if (args.Length < 2) { Console.WriteLine("[XUnitLogChecker]: The path to the log file and" + " the name of the wrapper are required for an" + " accurate check and fixing."); - return -1; + return MISSING_ARGS; } + // Creating variables for code clarity and ease of understanding. + string resultsDir = args[0]; string wrapperName = args[1]; - string tempLogName = $"{wrapperName}.templog.xml"; + string tempLogName = $"{wrapperName}.tempLog.xml"; string finalLogName = $"{wrapperName}.testResults.xml"; + string statsCsvName = $"{wrapperName}.testStats.csv"; string tempLogPath = Path.Combine(resultsDir, tempLogName); string finalLogPath = Path.Combine(resultsDir, finalLogName); + string statsCsvPath = Path.Combine(resultsDir, statsCsvName); - if (File.Exists(finalLogPath)) - { - Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did" - + " complete successfully!"); - return 0; - } + // If there is not even the temp log, then something went very badly with + // the work item in question. It will need a developer/engineer to look + // at it urgently. if (!File.Exists(tempLogPath)) { Console.WriteLine("[XUnitLogChecker]: No logs were found. Something" + " went very wrong with this item..."); - return -2; + + return SOMETHING_VERY_WRONG; + } + + // Read the stats csv file. + IEnumerable workItemStats = File.ReadLines(statsCsvPath); + + // The first value at the top of the csv represents the amount of tests + // that were expected to be run. + // + // NOTE: We know for certain the csv only includes numbers. Therefore, + // we're fine in using Int32.Parse() directly. + + int numExpectedTests = Int32.Parse(workItemStats.First().Split(',').First()); + + // The last line of the csv represents the status when the work item + // finished, successfully or not. It has the following format: + // (Tests Run, Tests Passed, Tests Failed, Tests Skipped) + + int[] workItemEndStatus = workItemStats.Last() + .Split(',') + .Select(x => Int32.Parse(x)) + .ToArray(); + + // If the final results log file is present, then we can assume everything + // went fine, and it's ready to go without any further processing. We just + // check the stats csv file to know how many tests were run, and display a + // brief summary of the work item. + + if (File.Exists(finalLogPath)) + { + Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did" + + " complete successfully!"); + + PrintWorkItemSummary(numExpectedTests, workItemEndStatus); + return SUCCESS; } + // Here goes the main core of the XUnit Log Checker :) + Console.WriteLine($"[XUnitLogChecker]: Item '{wrapperName}' did not" + + " finish running. Checking and fixing the log..."); + FixTheXml(tempLogPath); + PrintWorkItemSummary(numExpectedTests, workItemEndStatus); // Rename the temp log to the final log, so that Helix can use it without // knowing what transpired here. File.Move(tempLogPath, finalLogPath); - return 0; + return SUCCESS; + } + + static void PrintWorkItemSummary(int numExpectedTests, int[] workItemEndStatus) + { + Console.WriteLine($"\n{workItemEndStatus[0]}/{numExpectedTests} tests run."); + Console.WriteLine($"* {workItemEndStatus[1]} tests passed."); + Console.WriteLine($"* {workItemEndStatus[2]} tests failed."); + Console.WriteLine($"* {workItemEndStatus[3]} tests skipped.\n"); } static void FixTheXml(string xFile) { var tags = new Stack(); + // Flag to ensure we don't process tag-like-looking things while reading through + // a test's output. + bool inOutput = false; + foreach (string line in File.ReadLines(xFile)) { - // Get all XML tags found in the current line. - var opens = Regex.Matches(line, Patterns.OpenTag).ToList(); - var closes = Regex.Matches(line, Patterns.CloseTag).ToList(); + // Get all XML tags found in the current line and sort them in order + // of appearance. + Match[] opens = Regex.Matches(line, Patterns.OpenTag).ToArray(); + Match[] closes = Regex.Matches(line, Patterns.CloseTag).ToArray(); + TagResult[] allTags = GetOrderedTagMatches(opens, closes); - foreach (Match m in opens) + foreach (TagResult tr in allTags) { - // Push the next opening tag to the stack. We need only the actual - // tag without the opening and closing symbols, so we ask LINQ to - // lend us a hand. - string nextOpen = new String(m.Value - .Where(c => char.IsLetter(c)) - .ToArray()); - - // During testing, I encountered a case where one of the labels - // a 'test' tag, had incomplete quotes. Since this can only happen - // with 'test' tags, we handle this case separately. It's also - // worth noting that in theory, this case can only happen if it's - // the last line in the log. But to be safe, we'll check all the - // 'test' tags. - if (nextOpen.Equals("test")) - { - int quoteCount = 0; - - // Good ol' simple loop showed better performance in counting - // the quotes. - foreach (char c in line.AsSpan()) - { - if (c == '"') - quoteCount++; - } + // Get the name of the next tag. We need solely the text, so we + // ask LINQ to lend us a hand in removing the symbols from the string. + string tagText = new String(tr.Value.Where(c => char.IsLetter(c)).ToArray()); - if ((quoteCount % 2) != 0) - nextOpen = "testWithMissingQuote"; + // Found an opening tag. Push into the stack and move on to the next one. + if (tr.Category == TagCategory.OPENING && !inOutput) + { + tags.Push(tagText); + continue; } - tags.Push(nextOpen); - } - - // If we found any closing tags, then ensure they match their respective - // opening ones before continuing the analysis. - while (closes.Count > 0) - { - string nextClosing = closes.First().Value.Equals("]]>") - ? "CDATA" - : new String(closes.First() - .Value - .Where(c => char.IsLetter(c)) - .ToArray()); - - if (nextClosing.Equals(tags.Peek())) + // Found a closing tag. If we're currently in an output state, then + // check whether it's the output closing tag. Otherwise, ignore it. + // This is because in that case, it's just part of the output's text, + // rather than an actual XML log tag. + if (tr.Category == TagCategory.CLOSING) { - tags.Pop(); - closes.RemoveAt(0); } } } - if (tags.Count == 0) - { - Console.WriteLine($"[XUnitLogChecker]: XUnit log file '{xFile}' was A-OK!"); - } + Console.WriteLine("[XUnitLogChecker]: XUnit log file has been fixed!"); + } + + static TagResult[] GetOrderedTagMatches(Match[] openingTags, Match[] closingTags) + { + var result = new TagResult[openingTags.Length + closingTags.Length]; + int resIndex = opIndex = clIndex = 0; + + // Fill up the result array with the tags found, in order of appearance + // in the original log file line. + // + // As long as the result array hasn't been filled, then we know for certain + // that there's at least one unprocessed tag. - // Write the closings for all the lone opened tags we found. - using (StreamWriter xsw = File.AppendText(xFile)) - while (tags.Count > 0) + while (resIndex < result.Length) { - string tag = tags.Pop(); - if (tag.Equals("CDATA")) - xsw.WriteLine("]]>"); - else if (tag.Equals("testWithMissingQuote")) - xsw.WriteLine("\""); + if (opIndex < openingTags.Length) + { + // We still have pending tags on both lists, opening and closing. + // So, we add the one that appears first in the given log line. + if (clIndex < closingTags.Length) + { + if (openingTags[opIndex].Index < closingTags[clIndex].Index) + { + result[resIndex++] = new TagResult(openingTags[opIndex].Value, + TagCategory.OPENING); + opIndex++; + } + else + { + result[resIndex++] = new TagResult(closingTags[clIndex].Value, + TagCategory.CLOSING); + clIndex++; + } + } + + // Only opening tags remaining, so just add them. + else + { + result[resIndex++] = new TagResult(openingTags[opIndex].Value, + TagCategory.OPENING); + opIndex++; + } + } + + // Only closing tags remaining, so just add them. else - xsw.WriteLine($""); + { + result[resIndex++] = new TagResult(closingTags[clIndex].Value, + TagCategory.CLOSING); + clIndex++; + } } - - Console.WriteLine("[XUnitLogChecker]: XUnit log file has been fixed!"); + return result; } } From 2651eb198d0045ea4c17bd6f880a6028a40c048c Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Wed, 8 Feb 2023 18:00:18 -0800 Subject: [PATCH 17/21] XUnitLogChecker should be functional now. Ready to move back from draft to candidate PR. --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 75 +++++++++++++++++-- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index 7ec60cdf82903..f3acc45334cc1 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -9,8 +9,7 @@ public class XUnitLogChecker { private static class Patterns { - // public const string OpenTag = @"(\B<\w+)|(\B)|(\]\]>)"; } @@ -126,10 +125,12 @@ static void PrintWorkItemSummary(int numExpectedTests, int[] workItemEndStatus) static void FixTheXml(string xFile) { var tags = new Stack(); + string tagText = string.Empty; // Flag to ensure we don't process tag-like-looking things while reading through // a test's output. bool inOutput = false; + bool inCData = false; foreach (string line in File.ReadLines(xFile)) { @@ -141,13 +142,20 @@ static void FixTheXml(string xFile) foreach (TagResult tr in allTags) { - // Get the name of the next tag. We need solely the text, so we - // ask LINQ to lend us a hand in removing the symbols from the string. - string tagText = new String(tr.Value.Where(c => char.IsLetter(c)).ToArray()); - // Found an opening tag. Push into the stack and move on to the next one. - if (tr.Category == TagCategory.OPENING && !inOutput) + if (tr.Category == TagCategory.OPENING) { + // Get the name of the next tag. We need solely the text, so we + // ask LINQ to lend us a hand in removing the symbols from the string. + tagText = new String(tr.Value.Where(c => char.IsLetter(c)).ToArray()); + + // We are beginning to process a test's output. Set the flag to + // treat everything as such, until we get the closing output tag. + if (tagText.Equals("output") && !inOutput && !inCData) + inOutput = true; + else if (tagText.Equals("CDATA") && !inCData) + inCData = true; + tags.Push(tagText); continue; } @@ -158,17 +166,68 @@ static void FixTheXml(string xFile) // rather than an actual XML log tag. if (tr.Category == TagCategory.CLOSING) { + // As opposed to the usual XML tags we can find in the logs, + // the CDATA closing one doesn't have letters, so we treat it + // as a special case. + tagText = tr.Value.Equals("]]>") + ? "CDATA" + : new String(tr.Value + .Where(c => char.IsLetter(c)) + .ToArray()); + + if (inCData) + { + if (tagText.Equals("CDATA")) + { + tags.Pop(); + inCData = false; + } + else continue; + } + + if (inOutput) + { + if (tagText.Equals("output")) + { + tags.Pop(); + inOutput = false; + } + else continue; + } + + if (tagText.Equals(tags.Peek())) + { + tags.Pop(); + } } } } + if (tags.Count == 0) + { + Console.WriteLine($"[XUnitLogChecker]: XUnit log file '{xFile}' was A-OK!"); + } + + // Write the missing closings for all the opened tags we found. + using (StreamWriter xsw = File.AppendText(xFile)) + while (tags.Count > 0) + { + string tag = tags.Pop(); + if (tag.Equals("CDATA")) + xsw.WriteLine("]]>"); + else + xsw.WriteLine($""); + } + Console.WriteLine("[XUnitLogChecker]: XUnit log file has been fixed!"); } static TagResult[] GetOrderedTagMatches(Match[] openingTags, Match[] closingTags) { var result = new TagResult[openingTags.Length + closingTags.Length]; - int resIndex = opIndex = clIndex = 0; + int resIndex = 0; + int opIndex = 0; + int clIndex = 0; // Fill up the result array with the tags found, in order of appearance // in the original log file line. From 4fa4599b6ec6bf488b9ffe4f5af4ed0602800bf0 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Thu, 9 Feb 2023 11:12:29 -0800 Subject: [PATCH 18/21] Corrected changed signatures in the XHarnessTestRunner. --- .../XUnitWrapperGenerator.cs | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 14c38d092056c..678d37782dd2a 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -240,8 +240,12 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); builder.AppendLine("System.Console.SetOut(outputRecorder);"); - builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.templog.xml""))"); - builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.templog.xml"");"); + builder.Append("\n"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.xml""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.tempLog.xml"");"); + builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.testStats.csv""))"); + builder.AppendLine($@"System.IO.File.Delete(""{assemblyName}.testStats.csv"");"); + builder.Append("\n"); ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); @@ -249,11 +253,13 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI int testsLeftInCurrentTestExecutor = 0; int currentTestExecutor = 0; + // Open the stream writer for the temp log. + builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml""))"); + builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{"); + builder.AppendLine("statsCsvSw.WriteLine($\"{TestCount.Count},0,0,0\");"); + if (testInfos.Length > 0) { - // Open the stream writer for the temp log. - builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml"")) {{"); - // Break tests into groups of 50 so that we don't create an unreasonably large main method // Excessively large methods are known to take a long time to compile, and use excessive stack // leading to test failures. @@ -263,14 +269,24 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI { if (currentTestExecutor != 0) testExecutorBuilder.AppendLine("}"); + currentTestExecutor++; - testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(XUnitWrapperLibrary.TestSummary summary, XUnitWrapperLibrary.TestFilter filter, XUnitWrapperLibrary.TestOutputRecorder outputRecorder, System.Diagnostics.Stopwatch stopwatch, System.IO.StreamWriter tempLogSw){{"); - builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch, tempLogSw);"); + testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(" + + "XUnitWrapperLibrary.TestSummary summary, " + + "XUnitWrapperLibrary.TestFilter filter, " + + "XUnitWrapperLibrary.TestOutputRecorder outputRecorder, " + + "System.Diagnostics.Stopwatch stopwatch, " + + "System.IO.StreamWriter tempLogSw, " + + "System.IO.StreamWriter statsCsvSw){{"); + + builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch, tempLogSw, statsCsvSw);"); testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well } + testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter)); testsLeftInCurrentTestExecutor--; } + testExecutorBuilder.AppendLine("}"); builder.AppendLine("}"); } From b7e44b7f3fca9c3f863d251e8365eea22826a4aa Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Thu, 9 Feb 2023 16:23:55 -0800 Subject: [PATCH 19/21] Fixed a couple issues with browser-wasm generated XHarness test runners. --- .../Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 678d37782dd2a..816b567c2e653 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -256,7 +256,7 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI // Open the stream writer for the temp log. builder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}.templog.xml""))"); builder.AppendLine($@"using (System.IO.StreamWriter statsCsvSw = System.IO.File.AppendText(""{assemblyName}.testStats.csv"")){{"); - builder.AppendLine("statsCsvSw.WriteLine($\"{TestCount.Count},0,0,0\");"); + builder.AppendLine($"statsCsvSw.WriteLine(\"{testInfos.Length},0,0,0\");"); if (testInfos.Length > 0) { @@ -277,7 +277,7 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI + "XUnitWrapperLibrary.TestOutputRecorder outputRecorder, " + "System.Diagnostics.Stopwatch stopwatch, " + "System.IO.StreamWriter tempLogSw, " - + "System.IO.StreamWriter statsCsvSw){{"); + + "System.IO.StreamWriter statsCsvSw){"); builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch, tempLogSw, statsCsvSw);"); testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well From f342ede6b4d90279708a3894cfe39f417f29d97c Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Fri, 10 Feb 2023 13:39:11 -0800 Subject: [PATCH 20/21] Attempt to fix Browser-Wasm's weird way of layoutting tests. --- .../Common/XUnitLogChecker/XUnitLogChecker.cs | 19 ++++++++++++++++--- src/tests/Common/helixpublishwitharcade.proj | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs index f3acc45334cc1..119f70f00bffb 100644 --- a/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs +++ b/src/tests/Common/XUnitLogChecker/XUnitLogChecker.cs @@ -47,9 +47,21 @@ static int Main(string[] args) string resultsDir = args[0]; string wrapperName = args[1]; - string tempLogName = $"{wrapperName}.tempLog.xml"; - string finalLogName = $"{wrapperName}.testResults.xml"; - string statsCsvName = $"{wrapperName}.testStats.csv"; + // Browser-Wasm tests follow a different test files layout in Helix. + // Everything is located in a root folder, rather than the usual path + // with the wrapper name other platforms follow. + + string tempLogName = string.IsNullOrEmpty(wrapperName) + ? "tempLog.xml" + : $"{wrapperName}.tempLog.xml"; + + string finalLogName = string.IsNullOrEmpty(wrapperName) + ? "testResults.xml" + : $"{wrapperName}.testResults.xml"; + + string statsCsvName = string.IsNullOrEmpty(wrapperName) + ? "testStats.csv" + : $"{wrapperName}.testStats.csv"; string tempLogPath = Path.Combine(resultsDir, tempLogName); string finalLogPath = Path.Combine(resultsDir, finalLogName); @@ -63,6 +75,7 @@ static int Main(string[] args) { Console.WriteLine("[XUnitLogChecker]: No logs were found. Something" + " went very wrong with this item..."); + Console.WriteLine($"[XUnitLogChecker]: Expected log name: '{tempLogName}'"); return SOMETHING_VERY_WRONG; } diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index bb7f6134aa17c..a22c1fa0e2df7 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -482,7 +482,7 @@ dotnet %24HELIX_CORRELATION_PAYLOAD/XUnitLogChecker/ dotnet %25HELIX_CORRELATION_PAYLOAD%25/XUnitLogChecker/ - $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) $(_MergedWrapperName) + $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) '' From ec5aef038a8b551d50ea2b2f12ed97df58a1a227 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Date: Mon, 13 Feb 2023 16:51:55 -0800 Subject: [PATCH 21/21] Browser-Wasm is not possible to implement right here. Removing it... --- src/tests/Common/helixpublishwitharcade.proj | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index a22c1fa0e2df7..4880291225401 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -479,18 +479,11 @@ <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> - - dotnet %24HELIX_CORRELATION_PAYLOAD/XUnitLogChecker/ - dotnet %25HELIX_CORRELATION_PAYLOAD%25/XUnitLogChecker/ - $(XUnitLogCheckerCommand)XUnitLogChecker.dll $(_MergedWrapperRunScriptDirectoryRelative) '' - - -