diff --git a/src/coreclr/tools/r2rtest/BuildFolderSet.cs b/src/coreclr/tools/r2rtest/BuildFolderSet.cs index 26575746d6bbb5..80577a14be78cf 100644 --- a/src/coreclr/tools/r2rtest/BuildFolderSet.cs +++ b/src/coreclr/tools/r2rtest/BuildFolderSet.cs @@ -105,8 +105,6 @@ public bool Compile() Stopwatch stopwatch = Stopwatch.StartNew(); - ResolveTestExclusions(); - var compilationsToRun = new List(); foreach (BuildFolder folder in FoldersToBuild) @@ -549,19 +547,6 @@ public bool Build() return success; } - private void ResolveTestExclusions() - { - TestExclusionMap exclusions = TestExclusionMap.Create(_options); - foreach (BuildFolder folder in _buildFolders) - { - if (exclusions.TryGetIssue(folder.InputFolder, out string issueID)) - { - folder.IssueID = issueID; - continue; - } - } - } - private void AddBuildFolderExecutions(List executionsToRun, BuildFolder folder, int iterations) { foreach (ProcessInfo[][] execution in folder.Executions) diff --git a/src/coreclr/tools/r2rtest/CommandLineOptions.cs b/src/coreclr/tools/r2rtest/CommandLineOptions.cs index 3966dbed6ebcc1..82404653add802 100644 --- a/src/coreclr/tools/r2rtest/CommandLineOptions.cs +++ b/src/coreclr/tools/r2rtest/CommandLineOptions.cs @@ -56,7 +56,6 @@ void CreateCommand(string name, string description, Option[] options, Func Crossgen2JitPath { get; } = new("--crossgen2-jitpath") { Description = "Jit path to use for crossgen2" }; - public Option IssuesPath { get; } = - new("--issues-path", "-ip") { Description = "Path to issues.targets", Arity = ArgumentArity.ZeroOrMore }; - public Option CompilationTimeoutMinutes { get; } = new("--compilation-timeout-minutes", "-ct") { Description = "Compilation timeout (minutes)" }; @@ -337,7 +331,6 @@ public BuildOptions(R2RTestRootCommand cmd, ParseResult res) CompilationTimeoutMinutes = res.GetValue(cmd.CompilationTimeoutMinutes); ExecutionTimeoutMinutes = res.GetValue(cmd.ExecutionTimeoutMinutes); ReferencePath = res.GetValue(cmd.ReferencePath); - IssuesPath = res.GetValue(cmd.IssuesPath); R2RDumpPath = res.GetValue(cmd.R2RDumpPath); AspNetPath = res.GetValue(cmd.AspNetPath); MeasurePerf = res.GetValue(cmd.MeasurePerf); @@ -377,7 +370,6 @@ public BuildOptions(R2RTestRootCommand cmd, ParseResult res) public int CompilationTimeoutMinutes { get; } public int ExecutionTimeoutMinutes { get; } public DirectoryInfo[] ReferencePath { get; } - public FileInfo[] IssuesPath { get; } public FileInfo R2RDumpPath { get; } public DirectoryInfo AspNetPath { get; } public bool MeasurePerf { get; } diff --git a/src/coreclr/tools/r2rtest/TestExclusion.cs b/src/coreclr/tools/r2rtest/TestExclusion.cs deleted file mode 100644 index 36a90a6361081d..00000000000000 --- a/src/coreclr/tools/r2rtest/TestExclusion.cs +++ /dev/null @@ -1,229 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Runtime.InteropServices; -using System.Xml.Linq; - -using Microsoft.Build.Construction; -using Microsoft.Build.Evaluation; - -namespace R2RTest -{ - /// - /// This class represents a single test exclusion read from the issues.targets file. - /// - public class TestExclusion - { - /// - /// Path components (the individual directory levels read from the issues.targets file). - /// - public readonly string[] PathComponents; - - /// - /// True when an issues.targets exclusion spec ends with an '**'. - /// - public readonly bool OpenEnd; - - /// - /// Issue ID for the exclusion. - /// - public readonly string IssueID; - - /// - /// Initialize a test exclusion record read from the issues.targets file. - /// - /// Path components for this test exclusion - /// True when the entry ends with '**' - /// ID of the exclusion issue - public TestExclusion(string[] pathComponents, bool openEnd, string issueID) - { - PathComponents = pathComponents; - OpenEnd = openEnd; - IssueID = issueID; - } - - /// - /// Check whether the test exclusion entry matches a particular test folder / name. - /// - /// Components (directory levels) representing the test path - /// Index of first element in pathComponents to analyze - /// - public bool Matches(string[] pathComponents, int firstComponent) - { - if (pathComponents[firstComponent].Equals(PathComponents[0], StringComparison.OrdinalIgnoreCase) && - pathComponents.Length >= firstComponent + PathComponents.Length && - (OpenEnd || pathComponents.Length == firstComponent + PathComponents.Length)) - { - for (int matchIndex = 1; matchIndex < PathComponents.Length; matchIndex++) - { - if (!pathComponents[firstComponent + matchIndex].Equals(PathComponents[matchIndex], StringComparison.OrdinalIgnoreCase)) - { - return false; - } - } - return true; - } - return false; - } - } - - /// - /// Map of test exclusions with search acceleration. - /// - public class TestExclusionMap - { - public readonly Dictionary> _folderToExclusions; - - public TestExclusionMap() - { - _folderToExclusions = new Dictionary>(StringComparer.OrdinalIgnoreCase); - } - - /// - /// Add a single test exclusion to the map. - /// - /// - public void Add(TestExclusion exclusion) - { - if (!_folderToExclusions.TryGetValue(exclusion.PathComponents[0], out List exclusionsPerFolder)) - { - exclusionsPerFolder = new List(); - _folderToExclusions.Add(exclusion.PathComponents[0], exclusionsPerFolder); - } - exclusionsPerFolder.Add(exclusion); - } - - /// - /// Locate the issue ID for a given test path if it exists; return false when not. - /// - /// Path components representing the test path to check - /// Output issue ID when found, null otherwise - /// True when the test was found in the exclusion list, false otherwise - public bool TryGetIssue(string[] pathComponents, out string issueID) - { - for (int firstComponent = 0; firstComponent < pathComponents.Length; firstComponent++) - { - if (_folderToExclusions.TryGetValue(pathComponents[firstComponent], out List exclusions)) - { - foreach (TestExclusion exclusion in exclusions) - { - if (exclusion.Matches(pathComponents, firstComponent)) - { - issueID = exclusion.IssueID; - return true; - } - } - } - } - issueID = null; - return false; - } - - public bool TryGetIssue(string path, out string issueID) - { - string[] pathComponents = path.Split(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }); - return TryGetIssue(pathComponents, out issueID); - } - - private static XNamespace s_xmlNamespace = "http://schemas.microsoft.com/developer/msbuild/2003"; - - public static TestExclusionMap Create(BuildOptions options) - { - TestExclusionMap outputMap = new TestExclusionMap(); - - if (options.IssuesPath != null) - { - Dictionary> exclusionsByCondition = new Dictionary>(); - - foreach (FileInfo issuesProject in options.IssuesPath) - { - string issuesProjectPath = issuesProject.FullName; - XDocument issuesXml = XDocument.Load(issuesProjectPath); - foreach (XElement itemGroupElement in issuesXml.Root.Elements(s_xmlNamespace + "ItemGroup")) - { - string condition = itemGroupElement.Attribute("Condition")?.Value ?? ""; - List exclusions; - if (!exclusionsByCondition.TryGetValue(condition, out exclusions)) - { - exclusions = new List(); - exclusionsByCondition.Add(condition, exclusions); - } - foreach (XElement excludeListElement in itemGroupElement.Elements(s_xmlNamespace + "ExcludeList")) - { - string testPath = excludeListElement.Attribute("Include")?.Value ?? ""; - string issueID = excludeListElement.Element(s_xmlNamespace + "Issue")?.Value ?? "N/A"; - exclusions.Add(CreateTestExclusion(testPath, issueID)); - } - } - } - - Project project = new Project(); - project.SetGlobalProperty("XunitTestBinBase", "*"); - project.SetGlobalProperty("TargetArchitecture", "x64"); - project.SetGlobalProperty("RuntimeFlavor", "coreclr"); - // TODO: cross-OS CPAOT - project.SetGlobalProperty("TargetsWindows", (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "true" : "false")); - project.SetGlobalProperty("AltJitArch", "x64"); - project.SetGlobalProperty("RunTestViaIlLink", "false"); - - ProjectRootElement root = project.Xml; - root.AddTarget("GetListOfTestCmds"); - - ProjectPropertyGroupElement propertyGroup = root.AddPropertyGroup(); - - // Generate properties into the project to make it evaluate all conditions found in the targets file - List> testExclusionLists = new List>(); - testExclusionLists.Capacity = exclusionsByCondition.Count; - foreach (KeyValuePair> kvp in exclusionsByCondition) - { - string propertyName = "Condition_" + testExclusionLists.Count.ToString(); - bool emptyKey = string.IsNullOrEmpty(kvp.Key); - propertyGroup.AddProperty(propertyName, emptyKey ? "true" : "false"); - if (!emptyKey) - { - propertyGroup.AddProperty(propertyName, "true").Condition = kvp.Key; - } - testExclusionLists.Add(kvp.Value); - } - - project.Build(); - for (int exclusionListIndex = 0; exclusionListIndex < testExclusionLists.Count; exclusionListIndex++) - { - string conditionValue = project.GetProperty("Condition_" + exclusionListIndex.ToString()).EvaluatedValue; - if (conditionValue.Equals("true", StringComparison.OrdinalIgnoreCase)) - { - foreach (TestExclusion exclusion in testExclusionLists[exclusionListIndex]) - { - outputMap.Add(exclusion); - } - } - } - } - - return outputMap; - } - - private static TestExclusion CreateTestExclusion(string testPath, string issueId) - { - string[] pathComponents = testPath.Split(new char[] { '/' }); - int begin = 0; - if (begin < pathComponents.Length && pathComponents[begin] == "$(XunitTestBinBase)") - { - begin++; - } - int end = pathComponents.Length; - while (end > begin && (pathComponents[end - 1] == "*" || pathComponents[end - 1] == "**")) - { - end--; - } - bool openEnd = (end < pathComponents.Length && pathComponents[end] == "**"); - string[] outputComponents = new string[end - begin]; - Array.Copy(pathComponents, begin, outputComponents, 0, end - begin); - - return new TestExclusion(outputComponents, openEnd, issueId); - } - } -} diff --git a/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs b/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs deleted file mode 100644 index 254bef87ef6274..00000000000000 --- a/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Text; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -namespace TestExclusionListTasks; - -public class PatchExclusionListInApks : Task -{ - [Required] - public ITaskItem[]? ApkPaths { get; set; } - - [Required] - public ITaskItem[]? ExcludedTests { get; set; } - - public string? AndroidSdk { get; set; } - - public string? MinApiLevel { get; set; } - - public string? BuildToolsVersion { get; set; } - - public string? KeyStorePath { get; set; } - - public override bool Execute() - { - var apkBuilder = new ApkBuilder(Log); - apkBuilder.AndroidSdk = AndroidSdk; - apkBuilder.MinApiLevel = MinApiLevel; - apkBuilder.BuildToolsVersion = BuildToolsVersion; - apkBuilder.KeyStorePath = KeyStorePath; - - string testExclusionList = string.Join( - '\n', - (ExcludedTests ?? Enumerable.Empty()).Select(t => t.ItemSpec)); - foreach (ITaskItem apk in ApkPaths ?? Enumerable.Empty()) - { - string apkPath = apk.GetMetadata("FullPath")!; - apkBuilder.OutputDir = Path.GetDirectoryName(apkPath)!; - using (ZipArchive apkArchive = ZipFile.Open(apkPath, ZipArchiveMode.Update)) - { - ZipArchiveEntry assetsZipEntry = apkArchive.GetEntry("assets/assets.zip")!; - using ZipArchive assetsArchive = new ZipArchive(assetsZipEntry.Open(), ZipArchiveMode.Update); - ZipArchiveEntry testExclusionListEntry = assetsArchive.GetEntry("TestExclusionList.txt")!; - using StreamWriter textExclusionListWriter = new StreamWriter(testExclusionListEntry.Open()); - textExclusionListWriter.WriteLine(testExclusionList); - } - apkBuilder.ZipAndSignApk(apkPath); - } - return true; - } -} diff --git a/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj b/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj deleted file mode 100644 index 055b2ac3484c60..00000000000000 --- a/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - $(NetCoreAppToolCurrent) - $(NoWarn),CA1050,CA1850 - true - - - - - - - - - - - - - diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index 769ed407f8e20a..4ca37f0d06c0dd 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -552,7 +552,6 @@ $(IlasmRoundTripBashScript) $(SuperPMICollectionBashScript) # Allow test environment variables or precommands to override the ExePath ExePath=$(InputAssemblyName) -export TestExclusionListPath=$CORE_ROOT/TestExclusionList.txt $(BashEnvironmentVariables) # PreCommands $(CLRTestBashPreCommands) diff --git a/src/tests/Common/CLRTest.Execute.Batch.targets b/src/tests/Common/CLRTest.Execute.Batch.targets index 67a96674afd3e1..4b8d7ecfbdb3d9 100644 --- a/src/tests/Common/CLRTest.Execute.Batch.targets +++ b/src/tests/Common/CLRTest.Execute.Batch.targets @@ -426,7 +426,6 @@ $(SuperPMICollectionBatchScript) REM Allow test environment variables or precommands to override the ExePath set ExePath=$(InputAssemblyName) -set TestExclusionListPath=%CORE_ROOT%\TestExclusionList.txt REM Environment Variables $(BatchEnvironmentVariables) diff --git a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs index 8bce452f9a87c5..aaba120c8946a7 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs @@ -15,7 +15,6 @@ public sealed class GeneratedTestRunner : TestRunner private string _assemblyName; private TestFilter.ISearchClause? _filter; private Func _runTestsCallback; - private Dictionary _testExclusionTable; private readonly Boolean _writeBase64TestResults; @@ -23,13 +22,11 @@ public GeneratedTestRunner( LogWriter logger, Func runTestsCallback, string assemblyName, - Dictionary testExclusionTable, bool writeBase64TestResults) : base(logger) { _assemblyName = assemblyName; _runTestsCallback = runTestsCallback; - _testExclusionTable = testExclusionTable; _writeBase64TestResults = writeBase64TestResults; ResultsFileName = $"{_assemblyName}.testResults.xml"; @@ -41,7 +38,7 @@ public GeneratedTestRunner( public override Task Run(IEnumerable testAssemblies) { - LastTestRun = _runTestsCallback(new TestFilter(_filter, _testExclusionTable)); + LastTestRun = _runTestsCallback(new TestFilter(_filter)); PassedTests = LastTestRun.PassedTests; FailedTests = LastTestRun.FailedTests; SkippedTests = LastTestRun.SkippedTests; diff --git a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs index e2326154a13b3b..1c6350261e1b9d 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs @@ -13,28 +13,21 @@ public static async Task RunTests( Func runTestsCallback, string assemblyName, - string? filter, - Dictionary testExclusionTable) + string? filter) { - // If an exclusion list is passed as a filter, treat it as though no filter is provided here. - if (filter?.StartsWith("--exclusion-list=") == true) - { - filter = null; - } - ApplicationEntryPoint? entryPoint = null; if (OperatingSystem.IsAndroid()) { - entryPoint = new AndroidEntryPoint(new SimpleDevice(assemblyName), runTestsCallback, assemblyName, filter, testExclusionTable); + entryPoint = new AndroidEntryPoint(new SimpleDevice(assemblyName), runTestsCallback, assemblyName, filter); } if (OperatingSystem.IsMacCatalyst() || OperatingSystem.IsIOS() || OperatingSystem.IsTvOS()) { - entryPoint = new AppleEntryPoint(new SimpleDevice(assemblyName), runTestsCallback, assemblyName, filter, testExclusionTable); + entryPoint = new AppleEntryPoint(new SimpleDevice(assemblyName), runTestsCallback, assemblyName, filter); } if (OperatingSystem.IsBrowser() || OperatingSystem.IsWasi() ) { - entryPoint = new WasmEntryPoint(runTestsCallback, assemblyName, filter, testExclusionTable); + entryPoint = new WasmEntryPoint(runTestsCallback, assemblyName, filter); } if (entryPoint is null) { @@ -58,20 +51,17 @@ sealed class AppleEntryPoint : iOSApplicationEntryPointBase private readonly Func _runTestsCallback; private readonly string _assemblyName; private readonly string? _methodNameToRun; - private readonly Dictionary _testExclusionTable; public AppleEntryPoint( IDevice device, Func runTestsCallback, string assemblyName, - string? methodNameToRun, - Dictionary testExclusionTable) + string? methodNameToRun) { Device = device; _runTestsCallback = runTestsCallback; _assemblyName = assemblyName; _methodNameToRun = methodNameToRun; - _testExclusionTable = testExclusionTable; } protected override IDevice? Device { get; } @@ -79,7 +69,7 @@ public AppleEntryPoint( protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) { - var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionTable, writeBase64TestResults: false); + var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, writeBase64TestResults: false); if (_methodNameToRun is not null) { runner.SkipMethod(_methodNameToRun, isExcluded: false); @@ -96,20 +86,17 @@ sealed class AndroidEntryPoint : AndroidApplicationEntryPointBase private readonly Func _runTestsCallback; private readonly string _assemblyName; private readonly string? _methodNameToRun; - private readonly Dictionary _testExclusionTable; public AndroidEntryPoint( IDevice device, Func runTestsCallback, string assemblyName, - string? methodNameToRun, - Dictionary testExclusionTable) + string? methodNameToRun) { Device = device; _runTestsCallback = runTestsCallback; _assemblyName = assemblyName; _methodNameToRun = methodNameToRun; - _testExclusionTable = testExclusionTable; } protected override IDevice? Device { get; } @@ -117,7 +104,7 @@ public AndroidEntryPoint( protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) { - var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionTable, writeBase64TestResults: false); + var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, writeBase64TestResults: false); if (_methodNameToRun is not null) { runner.SkipMethod(_methodNameToRun, isExcluded: false); @@ -148,24 +135,21 @@ sealed class WasmEntryPoint : WasmApplicationEntryPointBase private readonly Func _runTestsCallback; private readonly string _assemblyName; private readonly string? _methodNameToRun; - private readonly Dictionary _testExclusionTable; public WasmEntryPoint( Func runTestsCallback, string assemblyName, - string? methodNameToRun, - Dictionary testExclusionTable) + string? methodNameToRun) { _runTestsCallback = runTestsCallback; _assemblyName = assemblyName; _methodNameToRun = methodNameToRun; - _testExclusionTable = testExclusionTable; } protected override int? MaxParallelThreads => 1; protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) { - var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, _testExclusionTable, writeBase64TestResults: true); + var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName, writeBase64TestResults: true); if (_methodNameToRun is not null) { runner.SkipMethod(_methodNameToRun, isExcluded: false); diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 4e79fc64f5a602..08702f531e39c7 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -181,7 +181,6 @@ public CodeBuilder GenerateTestExecution(ITestReporterWrapper testReporterWrappe using (builder.NewBracesScope()) { - builder.AppendLine("string reason = string.Empty;"); builder.AppendLine(testReporterWrapper.GenerateSkippedTestReporting(_innerTest)); } return builder; @@ -517,8 +516,6 @@ public CodeBuilder WrapTestExecutionWithReporting(CodeBuilder testExecutionExpre using (builder.NewBracesScope()) { - builder.AppendLine($"string reason = {_filterLocalIdentifier}" - + $".GetTestExclusionReason({test.TestNameExpression});"); builder.AppendLine(GenerateSkippedTestReporting(test)); } return builder; @@ -531,7 +528,7 @@ public string GenerateSkippedTestReporting(ITestInfo skippedTest) + $" \"{skippedTest.ContainingType}\"," + $" @\"{skippedTest.Method}\"," + $" System.TimeSpan.Zero," - + $" reason," + + $" string.Empty," + $" tempLogSw," + $" statsCsvSw);"; } diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 4162fea7438aa0..2b8e0c8e9a14d9 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -214,7 +214,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) .Where(data => { var (test, options) = data; - var filter = new XUnitWrapperLibrary.TestFilter(options.GlobalOptions.TestFilter(), null); + var filter = new XUnitWrapperLibrary.TestFilter(options.GlobalOptions.TestFilter()); return filter.ShouldRunTest($"{test.ContainingType}.{test.Method}", test.DisplayNameForFiltering, Array.Empty()); }) .Select((data, ct) => data.Left) @@ -309,10 +309,6 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos builder.AppendLine("void Initialize()"); using (builder.NewBracesScope()) { - builder.AppendLine("System.Collections.Generic.Dictionary testExclusionTable =" - + " XUnitWrapperLibrary.TestFilter.LoadTestExclusionTable();"); - builder.AppendLine(); - builder.AppendLine($@"if (System.IO.File.Exists(""{assemblyName}.tempLog.xml""))"); using (builder.NewBracesScope()) { @@ -326,7 +322,7 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos } builder.AppendLine(); - builder.AppendLine("filter = new (args, testExclusionTable);"); + builder.AppendLine("filter = new(args);"); builder.AppendLine("summary = new();"); builder.AppendLine("stopwatch = System.Diagnostics.Stopwatch.StartNew();"); builder.AppendLine("outputRecorder = new(System.Console.Out);"); @@ -465,13 +461,9 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI using (builder.NewBracesScope()) { - builder.AppendLine("System.Collections.Generic.Dictionary testExclusionTable =" - + " XUnitWrapperLibrary.TestFilter.LoadTestExclusionTable();"); - builder.AppendLine($@"return await XHarnessRunnerLibrary.RunnerEntryPoint.RunTests(RunTests," + $@" ""{assemblyName}""," - + $@" args.Length != 0 ? args[0] : null," - + $@" testExclusionTable);"); + + $@" args.Length != 0 ? args[0] : null);"); } builder.AppendLine("catch (System.Exception ex)"); diff --git a/src/tests/Common/XUnitWrapperLibrary/Help.cs b/src/tests/Common/XUnitWrapperLibrary/Help.cs index a395c8b8f534d5..a8a2099367204e 100644 --- a/src/tests/Common/XUnitWrapperLibrary/Help.cs +++ b/src/tests/Common/XUnitWrapperLibrary/Help.cs @@ -2,10 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; namespace XUnitWrapperLibrary; @@ -32,12 +28,6 @@ public static void WriteHelpText() Options: --help, -h, /?, -? Display this help text. - --exclusion-list=FILE - Path to a file containing a list of tests to exclude. - Each line in the file should be in the following format: - displayName, reason - (In the dotnet/runtime repository, this file is typically generated as part of the build process as 'src/tests/issues.targets'.) - If you are running tests outside the repository context, you may need to create your own exclusion list file in the format shown above. """; Console.WriteLine(help); diff --git a/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs b/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs index 39c7c33cc935b5..5bba14d71a3985 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs @@ -2,10 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; namespace XUnitWrapperLibrary; @@ -120,26 +116,16 @@ public override string ToString() private readonly ISearchClause? _filter; - // Test exclusion list is a compatibility measure allowing for a smooth migration - // away from the legacy issues.targets issue tracking system. Before we migrate - // all tests to the new model, it's easier to keep bug exclusions in the existing - // issues.targets file as a split model would be very confusing for developers - // and test monitors. - - // Explanation on the Test Exclusion Table is detailed in method LoadTestExclusionTable() - // later on in this file. - private readonly Dictionary? _testExclusionTable; - private readonly int _stripe; private readonly int _stripeCount = 1; private int _shouldRunQuery = -1; - public TestFilter(string? filterString, Dictionary? testExclusionTable) : - this(filterString == null ? Array.Empty() : new string[]{filterString}, testExclusionTable) + public TestFilter(string? filterString) : + this(filterString == null ? Array.Empty() : new string[]{filterString}) { } - public TestFilter(string[] filterArgs, Dictionary? testExclusionTable) + public TestFilter(string[] filterArgs) { string? filterString = null; @@ -173,23 +159,17 @@ public TestFilter(string[] filterArgs, Dictionary? testExclusion { _filter = ParseFilter(filterString, nameof(filterArgs)); } - _testExclusionTable = testExclusionTable; } - public TestFilter(ISearchClause? filter, Dictionary? testExclusionTable) + public TestFilter(ISearchClause? filter) { _filter = filter; - _testExclusionTable = testExclusionTable; } public bool ShouldRunTest(string fullyQualifiedName, string displayName, string[]? traits = null) { bool shouldRun; - if (_testExclusionTable is not null && _testExclusionTable.ContainsKey(displayName.Replace("\\", "/"))) - { - shouldRun = false; - } - else if (_filter is null) + if (_filter is null) { shouldRun = true; } @@ -206,84 +186,6 @@ public bool ShouldRunTest(string fullyQualifiedName, string displayName, string[ return false; } - public string GetTestExclusionReason(string testDisplayName) - { - if (_testExclusionTable is null) - return string.Empty; - - string trueDisplayName = testDisplayName.Replace("\\", "/"); - - return _testExclusionTable.ContainsKey(trueDisplayName) - ? _testExclusionTable[trueDisplayName] - : string.Empty; - } - - // GH dotnet/runtime issue #91562: Some tests are purposefully not run for a number - // of reasons. They are specified in src/tests/issues.targets, along with a brief - // explanation on why they are skipped inside an '' tag. - // - // When building any test or test subtree, if any of the tests built matches an entry - // in issues.targets, then the exclusions list file ($CORE_ROOT/TestExclusions.txt is - // the default) is updated by adding a new a comma-separated entry: - // - // 1) Test's Path (What is added to in issues.targets) - // 2) Reason for Skipping (What is written in the tag) - // - // When a test runner is executed (e.g. Methodical_d1.sh), it uses a compiler-generated - // source file to actually run the tests (e.g. FullRunner.g.cs - This is detailed in - // XUnitWrapperGenerator.cs). This generated source file is the one in charge of - // reading the test exclusions list file, and stores the comma-separated values into a - // table represented by the dictionary called _testExclusionTable in this file. - - public static Dictionary LoadTestExclusionTable() - { - Dictionary output = new Dictionary(); - - // Try reading the exclusion list as a base64-encoded semicolon-delimited string as a commmand-line arg. - string[] arguments = Environment.GetCommandLineArgs(); - string? testExclusionListArg = arguments.FirstOrDefault(arg => arg.StartsWith("--exclusion-list=")); - - if (!string.IsNullOrEmpty(testExclusionListArg)) - { - string testExclusionListPathFromCommandLine = testExclusionListArg.Substring("--exclusion-list=".Length); - ReadExclusionListToTable(testExclusionListPathFromCommandLine, output); - } - - // Try reading the exclusion list as a line-delimited file. - string? testExclusionListPath = Environment.GetEnvironmentVariable("TestExclusionListPath"); - - if (!string.IsNullOrEmpty(testExclusionListPath)) - { - ReadExclusionListToTable(testExclusionListPath, output); - } - return output; - } - - private static void ReadExclusionListToTable(string exclusionListPath, - Dictionary table) - { - IEnumerable excludedTestsWithReasons = File.ReadAllLines(exclusionListPath) - .Select(t => t.Split(',')); - - foreach (string[] testInfo in excludedTestsWithReasons) - { - // Each line read from the exclusion list file follows the following format: - // - // Test Path, Reason For Skipping - // - // This translates to the two-element arrays we are adding to the test - // exclusions table here. - - string testPath = testInfo[0]; - string skipReason = testInfo.Length > 1 ? testInfo[1] : string.Empty; - - if (!table.ContainsKey(testPath)) - { - table.Add(testPath, skipReason); - } - } - } - private static ISearchClause ParseFilter(string filterString, string argName) { ReadOnlySpan unsupported = ["|", "&", "(", ")", "!=", "!~"]; diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 7a9ff200f57fe1..a1f87b9290caf0 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -405,24 +405,10 @@ - - - <_ExcludeListProject Include="$(RepoRoot)src/tests/issues.targets"> - XunitTestBinBase=$(TestBinDir);$(_PropertiesToPass) - - - - - - - - - + DependsOnTargets="DiscoverMergedTestWrappers"> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) <_MergedWrapperName>$([System.Text.RegularExpressions.Regex]::Replace('%(_MergedWrapperMarker.FileName)', '\.(MergedTestAssembly(ForStress)?(\.\d+\.\d+)?|StandaloneTestRunner)$', '')) @@ -442,16 +428,12 @@ - + DependsOnTargets="DiscoverMergedTestWrappers"> <_MergedWrapperRunScript Include="@(_MergedWrapperMarker->'%(RootDir)%(Directory)$(MobileAppBundleDirName)/RunTests.$(TestScriptExtension)')" /> @@ -472,14 +454,9 @@ MakeRelative function calls Escape function internally that replaces all the Unicode characters with %. --> $([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)$(MobileAppBundleDirName)', %(FullPath))) - - <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> - <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> - - @@ -487,16 +464,12 @@ - - + DependsOnTargets="DiscoverMergedTestWrappers"> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) @@ -521,24 +494,17 @@ MakeRelative function calls Escape function internally that replaces all the Unicode characters with %. --> $([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)$(MobileAppBundleDirName)/$(_MergedWrapperName)/$(Configuration)-$(XCodeSdk)', %(FullPath))) - - <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)ExpectedExitCode.txt" Condition="Exists('$(_MergedWrapperDirectory)ExpectedExitCode.txt')" FileRelativeToPayloadsRootDirectory="ExpectedExitCode.txt" /> <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)$(_MergedWrapperName).XHarnessNoRunner" Condition="Exists('$(_MergedWrapperDirectory)$(_MergedWrapperName).XHarnessNoRunner')" FileRelativeToPayloadsRootDirectory="$(_MergedWrapperName).XHarnessNoRunner" /> - <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> - - + DependsOnTargets="DiscoverMergedTestWrappers"> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) @@ -555,16 +521,9 @@ <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)ExpectedExitCode.txt" Condition="Exists('$(_MergedWrapperDirectory)ExpectedExitCode.txt')" FileRelativeToPayloadsRootDirectory="ExpectedExitCode.txt" /> <_MergedPayloadFiles Include="$(_MergedWrapperDirectory)$(_MergedWrapperName).XHarnessNoRunner" Condition="Exists('$(_MergedWrapperDirectory)$(_MergedWrapperName).XHarnessNoRunner')" FileRelativeToPayloadsRootDirectory="$(_MergedWrapperName).XHarnessNoRunner" /> - - <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> - <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> - - - --arg=env:TestExclusionListPath=TestExclusionList.txt net.dot.%(ApkFileName) net.dot.MonoRunner $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - --set-env=TestExclusionListPath=TestExclusionList.txt $(AppleTestTarget) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) diff --git a/src/tests/Common/mergedrunnermobile.targets b/src/tests/Common/mergedrunnermobile.targets index b0ee8ea42617e6..cc054354103102 100644 --- a/src/tests/Common/mergedrunnermobile.targets +++ b/src/tests/Common/mergedrunnermobile.targets @@ -34,34 +34,6 @@ - - - - <_PlaceholderExclusionList Include="$(IntermediateOutputPath)/PlaceholderTestExclusionList.txt" /> - - - - - - <_ExpectedExitCodeFile Include="$(IntermediateOutputPath)/ExpectedExitCode.txt" /> - - - - - diff --git a/src/tests/Common/scripts/bringup_runtest.sh b/src/tests/Common/scripts/bringup_runtest.sh index fe6a84f7765d1e..21f273cdc040b2 100755 --- a/src/tests/Common/scripts/bringup_runtest.sh +++ b/src/tests/Common/scripts/bringup_runtest.sh @@ -499,7 +499,6 @@ function copy_test_native_bin_to_test_root { # Variables for unsupported and failing tests declare -a unsupportedTests declare -a failingTests -declare -a excludedTests declare -a playlistTests ((runFailingTestsOnly = 0)) @@ -534,15 +533,6 @@ function load_failing_tests { failingTests+=($(read_array "$(dirname "${BASH_SOURCE[0]}")/testsFailing.$ARCH.txt")) } -function load_excluded_tests { - # Read the exclusion file and populate the excludedTests array - while IFS=, read -r dllPath reasonMessage; do - # Extract the directory path from the dllPath and add it to the excludedTests array - dirPath=$(dirname "$dllPath") - excludedTests+=("$dirPath") - done < "${CORE_ROOT}/TestExclusionList.txt" -} - function load_playlist_tests { # Load the list of tests that are enabled as a part of this test playlist. playlistTests=($(read_array "${playlistFile}")) @@ -566,16 +556,6 @@ function is_failing_test { return 1 } -function is_excluded_test { - for excludedTest in "${excludedTests[@]}"; do - if [[ "$1" == "$excludedTest"* ]]; then - return 0 - fi - done - - return 1 -} - function is_playlist_test { for playlistTest in "${playlistTests[@]}"; do if [[ "$1" == "$playlistTest" ]]; then @@ -909,8 +889,6 @@ function start_test { skip_unsupported_test "$scriptFilePath" "$outputFilePath" & elif ((runFailingTestsOnly == 0)) && is_failing_test "$scriptFilePath"; then skip_failing_test "$scriptFilePath" "$outputFilePath" & - elif is_excluded_test "$scriptFilePath"; then - skip_unsupported_test "$scriptFilePath" "$outputFilePath" & else run_test "$scriptFilePath" "$outputFilePath" & fi @@ -1281,7 +1259,6 @@ then else load_unsupported_tests load_failing_tests - load_excluded_tests fi scriptPath=$(dirname $0) diff --git a/src/tests/Directory.Build.props b/src/tests/Directory.Build.props index c8aaf550a58d06..960236555d9533 100644 --- a/src/tests/Directory.Build.props +++ b/src/tests/Directory.Build.props @@ -173,8 +173,7 @@ true - + false diff --git a/src/tests/build.cmd b/src/tests/build.cmd index 5176b8ab9e3708..b2d7e5d102a61c 100644 --- a/src/tests/build.cmd +++ b/src/tests/build.cmd @@ -26,9 +26,6 @@ set "__RootBinDir=%__RepoRootDir%\artifacts" set "__LogsDir=%__RootBinDir%\log" set "__MsbuildDebugLogsDir=%__LogsDir%\MsbuildDebugLogs" -:: Default __Exclude to issues.targets -set __Exclude=%__RepoRootDir%\src\tests\issues.targets - REM __UnprocessedBuildArgs are args that we pass to msbuild (e.g. /p:TargetArchitecture=x64) set __commandName=%~nx0 set "__args= %*" @@ -122,7 +119,6 @@ if /i "%arg%" == "test" (set __BuildTestProject=!__BuildTestPro if /i "%arg%" == "dir" (set __BuildTestDir=!__BuildTestDir!%2%%3B&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%arg%" == "tree" (set __BuildTestTree=!__BuildTestTree!%2%%3B&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%arg%" == "log" (set __BuildLogRootName=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) -if /i "%arg%" == "exclude" (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%arg%" == "priority" (set __Priority=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%arg%" == "fsanitize" (set __CMakeArgs=%__CMakeArgs% "-DCLR_CMAKE_ENABLE_SANITIZERS=%2"&set __EnableNativeSanitizers=%2&set processedArgs=!processedArgs! %1=%2&shift&shift&goto Arg_Loop) @@ -157,7 +153,6 @@ if defined __TestArgParsing ( echo. echo.__BuildArch=%__BuildArch% echo.__BuildType=%__BuildType% - echo.__Exclude=%__Exclude% echo.__RebuildTests=%__RebuildTests% echo.__BuildTestProject=%__BuildTestProject% echo.__BuildTestDir=%__BuildTestDir% @@ -380,7 +375,6 @@ echo -Perfmap: Emit perfmap symbol files when compiling the framework assemblies echo -AllTargets: Build managed tests for all target platforms (including test projects in which CLRTestTargetUnsupported resolves to true). echo -ExcludeMonoFailures, Mono: Build the tests for the Mono runtime honoring mono-specific issues. echo. -echo -Exclude ^: Specify location of default exclusion file ^(defaults to tests\issues.targets if not specified^). echo Set to "" to disable default exclusion file. echo -Priority ^ : specify a set of tests that will be built and run, with priority N. echo 0: Build only priority 0 cases as essential testcases (default). diff --git a/src/tests/build.proj b/src/tests/build.proj index 79065c98a1d9c4..97abdc3f6d47ba 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -54,18 +54,11 @@ - - - False - True - - - - + @@ -300,11 +293,6 @@ AfterTargets="ManagedBuild;RestorePackages" Condition="'$(__CopyNativeTestBinaries)' != '1' and '$(__SkipGenerateLayout)' != '1' and !$(MonoAot) and !$(MonoFullAot)"> - - - - - - $(CORE_ROOT)\TestExclusionList.txt - - - - diff --git a/src/tests/build.sh b/src/tests/build.sh index aef00fe8149a02..a0709d7386517c 100755 --- a/src/tests/build.sh +++ b/src/tests/build.sh @@ -5,7 +5,6 @@ build_Tests() echo "${__MsgPrefix}Building Tests..." __ProjectFilesDir="$__TestDir" - __Exclude="$__RepoRootDir/src/tests/issues.targets" if [[ -f "${__TestBinDir}/build_info.json" ]]; then rm "${__TestBinDir}/build_info.json" @@ -107,7 +106,6 @@ build_Tests() export __MonoBinDir export __MsgPrefix export __ErrMsgPrefix - export __Exclude export EnableNativeSanitizers # Generate build command diff --git a/src/tests/issues.targets b/src/tests/issues.targets deleted file mode 100644 index dfd7f8a46b40a7..00000000000000 --- a/src/tests/issues.targets +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - -