[wasm] Add support for a random test case orderer, for xunit tests (d…

…otnet#65628)

* [wasm] Add support for a random test case orderer, for xunit tests

This is enabled by default for wasm with
`$(XUnitUseRandomizedTestOrderer)=true`.

When the library tests run, they print two messages like:

```
  info: Using random seed for test cases: 700149826
  info: Using random seed for collections: 700149826
  info: Starting:    System.Collections.Immutable.Tests.dll
```

These seeds are picked randomly every time the tests are run. To run the
tests with a specific seed, use environment variable
`XUNIT_RANDOM_ORDER_SEED`.

When running with tests, this can be used as:

    `WasmXHarnessMonoArgs="--setenv=XUNIT_RANDOM_ORDER_SEED=<seed>`

* Enable test orderer only for libraries tests

* [wasm] Automatically pass XUNIT_RANDOM_ORDER_SEED envvar to wasm apps

* Disable random test ordering for `System.Xml.RW.XmlWriterApi.Tests`

`System.Xml.Tests.TCCloseOutput.*` tests seem to depend on the order of
execution.

```
[06:35:14] fail: [FAIL] System.Xml.Tests.TCCloseOutput.CloseOutput_4(utils: XmlWriterUtils { Async = False, WriterType = UTF8Writer }, outputType: "Stream")
[06:35:14] info: System.IO.FileNotFoundException : File Not Found: writer.out
[06:35:14] info:    at XmlCoreTest.Common.FilePathUtil.getStream(String filename)
[06:35:14] info:    at System.Xml.Tests.TCCloseOutput.CloseOutput_4(XmlWriterUtils utils, String outputType)
[06:35:14] info:    at System.Reflection.RuntimeMethodInfo.InvokeWorker(Object obj, BindingFlags invokeAttr, Span`1 parameters)
```

* [wasm] Disable random tests ordering for `System.Runtime.Loader`

It seems to cause failures like, reproducible with
`XUNIT_RANDOM_ORDER_SEED=2106784294`:

```
[06:25:43] fail: [FAIL] System.Runtime.Loader.Tests.SatelliteAssembliesTests.SatelliteLoadsCorrectly_FromName(alc: "Empty", assemblyName: "System.Runtime.Loader.Tests", culture: "en")
[06:25:43] info: Assert.Same() Failure
[06:25:43] info: Expected: "Default" System.Runtime.Loader.DefaultAssemblyLoadContext #0
[06:25:43] info: Actual:   "Empty" System.Runtime.Loader.AssemblyLoadContext #4
[06:25:43] info:    at System.Runtime.Loader.Tests.SatelliteAssembliesTests.SatelliteLoadsCorrectly_FromName(String alc, String assemblyName, String culture)
[06:25:43] info:    at System.Reflection.RuntimeMethodInfo.InvokeWorker(Object obj, BindingFlags invokeAttr, Span`1 parameters)
```

* [wasm] Disable random test ordering for `System.Runtime.Numerics`

Randomized runs seem to fail with:

```
[03:29:54] info: Starting:    System.Runtime.Numerics.Tests.dll
[03:29:58] fail: [FAIL] System.Numerics.Tests.cast_toTest.RunDoubleExplicitCastToBigIntegerTests
[03:29:58] info: Assert.Equal() Failure
[03:29:58] info:           ↓ (pos 0)
[03:29:58] info: Expected: -0
[03:29:58] info: Actual:   0
[03:29:58] info:           ↑ (pos 0)
[03:29:58] info:    at System.Numerics.Tests.cast_toTest.VerifyDoubleExplicitCastToBigInteger(Double value)
[03:29:58] info:    at System.Numerics.Tests.cast_toTest.RunDoubleExplicitCastToBigIntegerTests()
[03:29:58] info:    at System.Reflection.RuntimeMethodInfo.InvokeWorker(Object obj, BindingFlags invokeAttr, Span`1 parameters)
```

Reproducible with `XUNIT_RANDOM_SEED_ORDER=1883302047`.

* Add issues for the failing tests

eng/testing/WasmRunnerTemplate.cmd

@@ -57,6 +57,10 @@ if [%XHARNESS_ARGS%] == [] (
     set "XHARNESS_ARGS=%JS_ENGINE% %JS_ENGINE_ARGS% %BROWSER_PATH% %MAIN_JS%"
 )
 
+if [%XUNIT_RANDOM_ORDER_SEED%] NEQ [] (
+    set "WasmXHarnessMonoArgs=%WasmXHarnessMonoArgs% --setenv=XUNIT_RANDOM_ORDER_SEED=%XUNIT_RANDOM_ORDER_SEED%"
+)
+
 echo EXECUTION_DIR=%EXECUTION_DIR%
 echo SCENARIO=%SCENARIO%
 echo XHARNESS_OUT=%XHARNESS_OUT%

eng/testing/WasmRunnerTemplate.sh

@@ -53,6 +53,10 @@ if [[ -z "$XHARNESS_ARGS" ]]; then
 	XHARNESS_ARGS="$JS_ENGINE $JS_ENGINE_ARGS $MAIN_JS"
 fi
 
+if [[ -n "$XUNIT_RANDOM_ORDER_SEED" ]]; then
+    WasmXHarnessMonoArgs="${WasmXHarnessMonoArgs} --setenv=XUNIT_RANDOM_ORDER_SEED=${XUNIT_RANDOM_ORDER_SEED}"
+fi
+
 echo EXECUTION_DIR=$EXECUTION_DIR
 echo SCENARIO=$SCENARIO
 echo XHARNESS_OUT=$XHARNESS_OUT
@@ -64,7 +68,6 @@ echo JS_ENGINE=$JS_ENGINE
 echo JS_ENGINE_ARGS=$JS_ENGINE_ARGS
 echo XHARNESS_ARGS=$XHARNESS_ARGS
 
-
 pushd $EXECUTION_DIR
 
 # ========================= BEGIN Test Execution ============================= 
@@ -83,4 +86,4 @@ echo ----- end $(date) ----- exit code $_exitCode ------------------------------
 
 echo "XHarness artifacts: $XHARNESS_OUT"
 
-exit $_exitCode
+exit $_exitCode

eng/testing/tests.mobile.targets

@@ -51,6 +51,10 @@
     <AdditionalXHarnessArguments Condition="'$(XUnitClassName)' != ''">$(AdditionalXHarnessArguments) -- -c=$(XUnitClassName)</AdditionalXHarnessArguments>
   </PropertyGroup>
 
+  <ItemGroup Condition="'$(XUnitUseRandomizedTestOrderer)' == 'true'">
+    <Compile Include="$(RepoRoot)src\libraries\Common\tests\Tests\RandomizedTestOrderAssemblyInfo.cs" />
+  </ItemGroup>
+
   <UsingTask Condition="'$(RunAOTCompilation)' == 'true'" TaskName="MonoAOTCompiler" AssemblyFile="$(MonoAOTCompilerTasksAssemblyPath)" />
   <Import Condition="'$(RunAOTCompilation)' == 'true'" Project="$(MonoAOTCompilerDir)MonoAOTCompiler.props" />
 

eng/testing/tests.wasm.targets

@@ -15,6 +15,7 @@
     <_ShellCommandSeparator Condition="'$(OS)' != 'Windows_NT'">&amp;&amp;</_ShellCommandSeparator>
     <WasmNativeStrip>false</WasmNativeStrip>
     <_WasmMainJSFileName Condition="'$(WasmMainJSPath)' != ''">$([System.IO.Path]::GetFileName('$(WasmMainJSPath)'))</_WasmMainJSFileName>
+    <XUnitUseRandomizedTestOrderer Condition="'$(XUnitUseRandomizedTestOrderer)' == '' and '$(IsTestProject)' == 'true'">true</XUnitUseRandomizedTestOrderer>
   </PropertyGroup>
 
   <PropertyGroup>

src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs

@@ -0,0 +1,77 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Threading;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+#nullable enable
+
+namespace TestUtilities;
+
+// Based on https://github.com/xunit/xunit/blob/v2/src/xunit.execution/Sdk/DefaultTestCaseOrderer.cs
+
+public class RandomTestCaseOrderer : ITestCaseOrderer
+{
+    public const string RandomSeedEnvironmentVariableName = "XUNIT_RANDOM_ORDER_SEED";
+
+    public static readonly Lazy<int> LazySeed = new (GetSeed, LazyThreadSafetyMode.ExecutionAndPublication);
+    private readonly IMessageSink _diagnosticMessageSink;
+
+    private static int GetSeed()
+    {
+        string? seedEnvVar = Environment.GetEnvironmentVariable(RandomSeedEnvironmentVariableName);
+        if (string.IsNullOrEmpty(seedEnvVar) || !int.TryParse(seedEnvVar, out int seed))
+        {
+            seed = new Random().Next();
+        }
+
+        return seed;
+    }
+
+    public RandomTestCaseOrderer(IMessageSink diagnosticMessageSink)
+    {
+        diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using random seed for test cases: {LazySeed.Value}"));
+        _diagnosticMessageSink = diagnosticMessageSink;
+    }
+
+    public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases) where TTestCase : ITestCase
+        => TryRandomize(testCases.ToList(), _diagnosticMessageSink, out List<TTestCase>? randomizedTests)
+                    ? randomizedTests
+                    : testCases;
+
+    public static bool TryRandomize<T>(List<T> tests, IMessageSink messageSink, [NotNullWhen(true)] out List<T>? randomizedTests)
+    {
+        randomizedTests = null;
+        try
+        {
+            randomizedTests = Randomize(tests.ToList());
+            return true;
+        }
+        catch (Exception ex)
+        {
+            messageSink.OnMessage(new DiagnosticMessage($"Failed to randomize test cases: {ex}"));
+            return false;
+        }
+
+        static List<T> Randomize(List<T> tests)
+        {
+            var result = new List<T>(tests.Count);
+
+            var randomizer = new Random(LazySeed.Value);
+
+            while (tests.Count > 0)
+            {
+                int next = randomizer.Next(tests.Count);
+                result.Add(tests[next]);
+                tests.RemoveAt(next);
+            }
+
+            return result;
+        }
+    }
+}

src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs

@@ -0,0 +1,29 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using Xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+#nullable enable
+
+namespace TestUtilities;
+
+public class RandomTestCollectionOrderer : ITestCollectionOrderer
+{
+    private readonly IMessageSink _diagnosticMessageSink;
+
+    public RandomTestCollectionOrderer(IMessageSink diagnosticMessageSink)
+    {
+        diagnosticMessageSink.OnMessage(new DiagnosticMessage(
+                                                $"Using random seed for collections: {RandomTestCaseOrderer.LazySeed.Value}"));
+        _diagnosticMessageSink = diagnosticMessageSink;
+    }
+
+    public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollection> testCollections)
+        => RandomTestCaseOrderer.TryRandomize(testCollections.ToList(), _diagnosticMessageSink, out List<ITestCollection>? randomizedTests)
+                    ? randomizedTests
+                    : testCollections;
+}

src/libraries/Common/tests/TestUtilities/TestUtilities.csproj

@@ -43,6 +43,9 @@
       variant from the Common folder and adding the missing members manually.
     -->
     <Compile Include="Interop\Interop.Libraries.cs" />
+
+    <Compile Include="RandomTestCaseOrderer.cs" />
+    <Compile Include="RandomTestCollectionOrderer.cs" />
   </ItemGroup>
   <!-- Windows imports -->
   <ItemGroup>

src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs

@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit;
+
+[assembly: TestCaseOrderer("TestUtilities.RandomTestCaseOrderer", "TestUtilities")]
+[assembly: TestCollectionOrderer("TestUtilities.RandomTestCollectionOrderer", "TestUtilities")]

src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
+    <XUnitUseRandomizedTestOrderer>false</XUnitUseRandomizedTestOrderer>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="EndOfLineHandlingTests.cs" />

src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj

@@ -8,6 +8,9 @@
     <GenerateDependencyFile>false</GenerateDependencyFile>
     <!-- EnC tests on targets without a remote executor need the environment variable set before launching the test -->
     <WasmXHarnessMonoArgs>--setenv=DOTNET_MODIFIABLE_ASSEMBLIES=debug</WasmXHarnessMonoArgs>
+
+    <!-- disabled due to https://github.com/dotnet/runtime/issues/65672 -->
+    <XUnitUseRandomizedTestOrderer>false</XUnitUseRandomizedTestOrderer>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="ApplyUpdateTest.cs" />

src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj

@@ -2,6 +2,9 @@
   <PropertyGroup>
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
     <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
+
+    <!-- disabled due to https://github.com/dotnet/runtime/issues/65671 -->
+    <XUnitUseRandomizedTestOrderer>false</XUnitUseRandomizedTestOrderer>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="BigInteger\absolutevalue.cs" />
@@ -51,4 +54,4 @@
     <Compile Include="BigInteger\TryWriteBytes.cs" />
     <Compile Include="ComplexTests.cs" />
   </ItemGroup>
-</Project>
+</Project>