From bd08a682e645504960e9be62fec2ca6b4d5d0910 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Fri, 3 Feb 2023 12:05:31 -0800 Subject: [PATCH 01/13] Add Linux NativeAOT debug info regression test The test runs llvm-dwarfdump and verifies that the number of warnings and errors is expected. As we improve the debug information we can start tracking individual warnings and errors, hopefully bringing this number to zero. --- .../SmokeTests/DwarfDump/DwarfDump.csproj | 12 +++++ .../nativeaot/SmokeTests/DwarfDump/Program.cs | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj create mode 100644 src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj new file mode 100644 index 0000000000000..974ccda7f0ba9 --- /dev/null +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj @@ -0,0 +1,12 @@ + + + + Exe + BuildAndRun + 0 + + + + + + diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs new file mode 100644 index 0000000000000..2b5608e3615be --- /dev/null +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -0,0 +1,44 @@ +using System; +using System.Diagnostics; + +public class Program +{ + public static int Main(string[] args) + { + if (!OperatingSystem.IsLinux()) + { + // Linux-only test + return 100; + } + + var proc = Process.Start(new ProcessStartInfo + { + FileName = "/bin/sh", + Arguments = $"-c \"llvm-dwarfdump --verify {Environment.ProcessPath}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }); + + // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list + const int ExpectedCount = 13396; + int count = 0; + string line; + while ((line = proc.StandardOutput.ReadLine()) != null) + { + if (line.Contains("warning:") || line.Contains("error:")) + { + count++; + } + } + proc.WaitForExit(); + if (count != ExpectedCount) + { + Console.WriteLine($"Found {count} warnings and errors, expected {ExpectedCount}"); + Console.WriteLine("This is likely a result of debug info changes. To see the new output, run the following command:"); + Console.WriteLine("\tllvm-dwarfdump --verify " + Environment.ProcessPath); + return 1; + } + return 100; + } +} \ No newline at end of file From 9c0ee5eaa5ba7262c94e144da32b91df83be9c23 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Fri, 3 Feb 2023 12:06:51 -0800 Subject: [PATCH 02/13] Check exit code --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index 2b5608e3615be..31a03d2ac503d 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -32,6 +32,12 @@ public static int Main(string[] args) } } proc.WaitForExit(); + if (proc.ExitCode != 0) + { + Console.WriteLine("llvm-dwarfdump failed with exit code " + proc.ExitCode); + Console.WriteLine(proc.StandardError.ReadToEnd()); + return 1; + } if (count != ExpectedCount) { Console.WriteLine($"Found {count} warnings and errors, expected {ExpectedCount}"); From 81a91d54a0d8e08fb6462a2fc4ba7364408b9d4d Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Fri, 3 Feb 2023 12:51:28 -0800 Subject: [PATCH 03/13] Don't assert llvm-dwarfdump succeeded --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index 31a03d2ac503d..574b6ca99f20f 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -20,6 +20,12 @@ public static int Main(string[] args) UseShellExecute = false }); + if (proc is null) + { + Console.WriteLine("llvm-dwarfdump could not run"); + return 1; + } + // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list const int ExpectedCount = 13396; int count = 0; @@ -32,12 +38,6 @@ public static int Main(string[] args) } } proc.WaitForExit(); - if (proc.ExitCode != 0) - { - Console.WriteLine("llvm-dwarfdump failed with exit code " + proc.ExitCode); - Console.WriteLine(proc.StandardError.ReadToEnd()); - return 1; - } if (count != ExpectedCount) { Console.WriteLine($"Found {count} warnings and errors, expected {ExpectedCount}"); From 1b3d122cc19e8e708ddd8df720dc57ac27820546 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Fri, 3 Feb 2023 15:33:41 -0800 Subject: [PATCH 04/13] Only run in Debug+nativeaot configuration, and change baseline --- src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj | 2 ++ src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj index 974ccda7f0ba9..518bfe2478a26 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj @@ -4,6 +4,8 @@ Exe BuildAndRun 0 + + true diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index 574b6ca99f20f..fd3e6aeb63538 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -27,7 +27,7 @@ public static int Main(string[] args) } // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list - const int ExpectedCount = 13396; + const int ExpectedCount = 23126; int count = 0; string line; while ((line = proc.StandardOutput.ReadLine()) != null) From 00be9a084dc876d73b6d56ed9a6abbe66894f3ff Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Mon, 6 Feb 2023 22:20:36 -0800 Subject: [PATCH 05/13] Change expected warning count and print llvm-dwarfdump version --- .../nativeaot/SmokeTests/DwarfDump/Program.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index fd3e6aeb63538..dfec4ca537881 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -11,7 +11,22 @@ public static int Main(string[] args) return 100; } - var proc = Process.Start(new ProcessStartInfo + Console.WriteLine("Running llvm-dwarfdump"); + var proc = Process.Start("llvm-dwarfdump", "--version"); + + if (proc is null) + { + Console.WriteLine("llvm-dwarfdump could not run"); + return 1; + } + + proc.WaitForExit(); + if (proc.ExitCode != 0) + { + return 2; + } + + proc = Process.Start(new ProcessStartInfo { FileName = "/bin/sh", Arguments = $"-c \"llvm-dwarfdump --verify {Environment.ProcessPath}", @@ -20,14 +35,8 @@ public static int Main(string[] args) UseShellExecute = false }); - if (proc is null) - { - Console.WriteLine("llvm-dwarfdump could not run"); - return 1; - } - // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list - const int ExpectedCount = 23126; + const int ExpectedCount = 46253; int count = 0; string line; while ((line = proc.StandardOutput.ReadLine()) != null) From c39f3f14fd61ccace2e664711297085ff43a4193 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Wed, 8 Feb 2023 15:22:57 -0800 Subject: [PATCH 06/13] Use llvm-dwarfdump from the JIT tools NuGet package --- .../nativeaot/SmokeTests/DwarfDump/Program.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index dfec4ca537881..c08171fb7000a 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -1,35 +1,44 @@ using System; using System.Diagnostics; +using System.Runtime.InteropServices; public class Program { public static int Main(string[] args) { - if (!OperatingSystem.IsLinux()) + var llvmDwarfDumpPath = Path.Combine( + Environment.GetEnvironmentVariable("CORE_ROOT"), + "runtimes", + RuntimeInformation.RuntimeIdentifier, + "native", + "llvm-dwarfdump"); + + if (!File.Exists(llvmDwarfDumpPath)) { // Linux-only test - return 100; + Console.WriteLine("Could not find llvm-dwarfdump at path: " + llvmDwarfDumpPath); + return 1; } Console.WriteLine("Running llvm-dwarfdump"); - var proc = Process.Start("llvm-dwarfdump", "--version"); + var proc = Process.Start(llvmDwarfDumpPath, "--version"); if (proc is null) { Console.WriteLine("llvm-dwarfdump could not run"); - return 1; + return 2; } proc.WaitForExit(); if (proc.ExitCode != 0) { - return 2; + return 3; } proc = Process.Start(new ProcessStartInfo { FileName = "/bin/sh", - Arguments = $"-c \"llvm-dwarfdump --verify {Environment.ProcessPath}", + Arguments = $"-c \"{llvmDwarfDumpPath} --verify {Environment.ProcessPath}", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false From 52e235e1cdd92f88b77b23c26363fa292ef92ef4 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Sun, 12 Feb 2023 20:31:03 -0800 Subject: [PATCH 07/13] Add using --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index c08171fb7000a..8c7014147aed9 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; public class Program From 0218a788b9e4c5521986ad68ca4a7aed372d071c Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Sun, 12 Feb 2023 20:48:28 -0800 Subject: [PATCH 08/13] Use a range for warnings instead of hardcoding an exact number --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index 8c7014147aed9..a6b6d39522165 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -9,6 +9,7 @@ public static int Main(string[] args) { var llvmDwarfDumpPath = Path.Combine( Environment.GetEnvironmentVariable("CORE_ROOT"), + "SuperFileCheck", "runtimes", RuntimeInformation.RuntimeIdentifier, "native", @@ -46,7 +47,8 @@ public static int Main(string[] args) }); // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list - const int ExpectedCount = 46253; + const int MinWarnings = 18400; + const int MaxWarnings = 48600; int count = 0; string line; while ((line = proc.StandardOutput.ReadLine()) != null) @@ -57,9 +59,9 @@ public static int Main(string[] args) } } proc.WaitForExit(); - if (count != ExpectedCount) + if (count is not >= MinWarnings and <= MaxWarnings) { - Console.WriteLine($"Found {count} warnings and errors, expected {ExpectedCount}"); + Console.WriteLine($"Found {count} warnings and errors, expected between {MinWarnings} and {MaxWarnings}"); Console.WriteLine("This is likely a result of debug info changes. To see the new output, run the following command:"); Console.WriteLine("\tllvm-dwarfdump --verify " + Environment.ProcessPath); return 1; From 88f56f24eb092e4474c9cd514047cdd8ecd55eb2 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Sun, 12 Feb 2023 21:44:57 -0800 Subject: [PATCH 09/13] Skip if not Linux --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index a6b6d39522165..7e2cc8d09b550 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -7,6 +7,12 @@ public class Program { public static int Main(string[] args) { + if (!runtimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + // Linux-only test + return 100; + } + var llvmDwarfDumpPath = Path.Combine( Environment.GetEnvironmentVariable("CORE_ROOT"), "SuperFileCheck", From f478e1b3af620d596307bb1647851eccc6b406eb Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Mon, 13 Feb 2023 10:00:30 -0800 Subject: [PATCH 10/13] Update src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michal Strehovský --- src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj index 518bfe2478a26..5ce86e7d8fcc0 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj @@ -5,7 +5,7 @@ BuildAndRun 0 - true + true From 2068cecef0e384cf23a4394031ab926c8f23388e Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Mon, 13 Feb 2023 10:01:06 -0800 Subject: [PATCH 11/13] Remove manual OS skip --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index 7e2cc8d09b550..a7a8192550f8f 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -7,12 +7,6 @@ public class Program { public static int Main(string[] args) { - if (!runtimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - // Linux-only test - return 100; - } - var llvmDwarfDumpPath = Path.Combine( Environment.GetEnvironmentVariable("CORE_ROOT"), "SuperFileCheck", @@ -74,4 +68,4 @@ public static int Main(string[] args) } return 100; } -} \ No newline at end of file +} From 1bf9c6555e5f3cc9374252257e1e634ff42fe234 Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Mon, 13 Feb 2023 20:34:05 -0800 Subject: [PATCH 12/13] Update src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michal Strehovský --- src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj index 5ce86e7d8fcc0..783b9db6cfc69 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/DwarfDump.csproj @@ -5,7 +5,7 @@ BuildAndRun 0 - true + true From 906ba3746a07bb31ae6cee87fa3c47872b6aed5f Mon Sep 17 00:00:00 2001 From: Andy Gocke Date: Tue, 14 Feb 2023 09:56:10 -0800 Subject: [PATCH 13/13] PR feedback --- src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs index a7a8192550f8f..086bb995155ac 100644 --- a/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs +++ b/src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs @@ -1,3 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; using System.Diagnostics; using System.IO; @@ -39,8 +42,8 @@ public static int Main(string[] args) proc = Process.Start(new ProcessStartInfo { - FileName = "/bin/sh", - Arguments = $"-c \"{llvmDwarfDumpPath} --verify {Environment.ProcessPath}", + FileName = llvmDwarfDumpPath, + Arguments = $"--verify {Environment.ProcessPath}", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false @@ -59,12 +62,13 @@ public static int Main(string[] args) } } proc.WaitForExit(); + Console.WriteLine($"Found {count} warnings and errors"); if (count is not >= MinWarnings and <= MaxWarnings) { Console.WriteLine($"Found {count} warnings and errors, expected between {MinWarnings} and {MaxWarnings}"); Console.WriteLine("This is likely a result of debug info changes. To see the new output, run the following command:"); Console.WriteLine("\tllvm-dwarfdump --verify " + Environment.ProcessPath); - return 1; + return 10; } return 100; }