perf: reduce lock contention in test discovery and scheduling

TUnit.Engine.Tests/Scheduling/ConstraintKeySchedulerConcurrencyTests.cs

@@ -0,0 +1,35 @@
+using Shouldly;
+using TUnit.Engine.Tests.Enums;
+
+namespace TUnit.Engine.Tests.Scheduling;
+
+/// <summary>
+/// Engine tests that validate ConstraintKeyScheduler handles high contention scenarios
+/// without deadlocks. Invokes TUnit.TestProject.ConstraintKeyStressTests to ensure:
+/// 1. Many concurrent tests with overlapping constraint keys complete successfully
+/// 2. No deadlocks occur under high contention
+/// 3. Tests complete within reasonable timeout
+/// </summary>
+public class ConstraintKeySchedulerConcurrencyTests(TestMode testMode) : InvokableTestBase(testMode)
+{
+    [Test]
+    [Repeat(5)]
+    public async Task HighContention_WithOverlappingConstraints_CompletesWithoutDeadlock()
+    {
+        // This test establishes baseline behavior before optimization
+        // It runs tests with overlapping constraint keys to verify the scheduler
+        // can handle high contention without deadlocking
+
+        // Timeout proves no deadlock occurred - if tests complete before timeout, scheduler is working correctly
+        using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+        await RunTestsWithFilter("/*/*/ConstraintKeyStressTests/*",
+        [
+            result => result.ResultSummary.Outcome.ShouldBe("Completed"),
+            result => result.ResultSummary.Counters.Total.ShouldBe(30), // 10 tests × 3 runs (Repeat(2))
+            result => result.ResultSummary.Counters.Passed.ShouldBe(30),
+            result => result.ResultSummary.Counters.Failed.ShouldBe(0)
+        ],
+        new RunOptions().WithForcefulCancellationToken(cts.Token));
+    }
+}

TUnit.Engine/Discovery/ReflectionTestDataCollector.cs

@@ -1,5 +1,6 @@
 using System.Buffers;
 using System.Collections.Concurrent;
+using System.Collections.Immutable;
 using System.Diagnostics.CodeAnalysis;
 using System.Reflection;
 using System.Runtime.CompilerServices;
@@ -28,8 +29,7 @@ namespace TUnit.Engine.Discovery;
 internal sealed class ReflectionTestDataCollector : ITestDataCollector
 {
     private static readonly ConcurrentDictionary<Assembly, bool> _scannedAssemblies = new();
-    private static readonly List<TestMetadata> _discoveredTests = new(capacity: 1000); // Pre-sized for typical test suites
-    private static readonly Lock _discoveredTestsLock = new(); // Lock for thread-safe access to _discoveredTests
+    private static ImmutableList<TestMetadata> _discoveredTests = ImmutableList<TestMetadata>.Empty;
     private static readonly ConcurrentDictionary<Assembly, Type[]> _assemblyTypesCache = new();
     private static readonly ConcurrentDictionary<Type, MethodInfo[]> _typeMethodsCache = new();
 
@@ -47,10 +47,7 @@ private static Assembly[] GetCachedAssemblies()
     public static void ClearCaches()
     {
         _scannedAssemblies.Clear();
-        lock (_discoveredTestsLock)
-        {
-            _discoveredTests.Clear();
-        }
+        Interlocked.Exchange(ref _discoveredTests, ImmutableList<TestMetadata>.Empty);
         _assemblyTypesCache.Clear();
         _typeMethodsCache.Clear();
         lock (_assemblyCacheLock)
@@ -133,12 +130,15 @@ public async Task<IEnumerable<TestMetadata>> CollectTestsAsync(string testSessio
         var dynamicTests = await DiscoverDynamicTests(testSessionId).ConfigureAwait(false);
         newTests.AddRange(dynamicTests);
 
-        // Add to discovered tests with lock (better enumeration performance than ConcurrentBag)
-        lock (_discoveredTestsLock)
+        // Atomic swap - no lock needed for readers
+        ImmutableList<TestMetadata> original, updated;
+        do
         {
-            _discoveredTests.AddRange(newTests);
-            return new List<TestMetadata>(_discoveredTests);
-        }
+            original = _discoveredTests;
+            updated = original.AddRange(newTests);
+        } while (Interlocked.CompareExchange(ref _discoveredTests, updated, original) != original);
+
+        return _discoveredTests;
     }
 
     public async IAsyncEnumerable<TestMetadata> CollectTestsStreamingAsync(
@@ -171,21 +171,15 @@ public async IAsyncEnumerable<TestMetadata> CollectTestsStreamingAsync(
             // Stream tests from this assembly
             await foreach (var test in DiscoverTestsInAssemblyStreamingAsync(assembly, cancellationToken))
             {
-                lock (_discoveredTestsLock)
-                {
-                    _discoveredTests.Add(test);
-                }
+                ImmutableInterlocked.Update(ref _discoveredTests, list => list.Add(test));
                 yield return test;
             }
         }
 
         // Stream dynamic tests
         await foreach (var dynamicTest in DiscoverDynamicTestsStreamingAsync(testSessionId, cancellationToken))
         {
-            lock (_discoveredTestsLock)
-            {
-                _discoveredTests.Add(dynamicTest);
-            }
+            ImmutableInterlocked.Update(ref _discoveredTests, list => list.Add(dynamicTest));
             yield return dynamicTest;
         }
     }

TUnit.Engine/Scheduling/ConstraintKeyScheduler.cs

@@ -55,15 +55,24 @@ public async ValueTask ExecuteTestsWithConstraintsAsync(
             bool canStart;
             lock (lockObject)
             {
-                // Check if all constraint keys are available
-                canStart = !constraintKeys.Any(key => lockedKeys.Contains(key));
-
+                // Check if all constraint keys are available - manual loop avoids LINQ allocation
+                canStart = true;
+                var keyCount = constraintKeys.Count;
+                for (var i = 0; i < keyCount; i++)
+                {
+                    if (lockedKeys.Contains(constraintKeys[i]))
+                    {
+                        canStart = false;
+                        break;
+                    }
+                }
+
                 if (canStart)
                 {
                     // Lock all the constraint keys for this test
-                    foreach (var key in constraintKeys)
+                    for (var i = 0; i < keyCount; i++)
                     {
-                        lockedKeys.Add(key);
+                        lockedKeys.Add(constraintKeys[i]);
                     }
                 }
             }
@@ -146,44 +155,54 @@ private async Task ExecuteTestAndReleaseKeysAsync(
             parallelLimiterSemaphore?.Release();
 
             // Release the constraint keys and check if any waiting tests can now run
+            // Pre-allocate lists outside the lock to minimize lock duration
             var testsToStart = new List<(AbstractExecutableTest Test, IReadOnlyList<string> ConstraintKeys, TaskCompletionSource<bool> StartSignal)>();
-
+            var testsToRequeue = new List<(AbstractExecutableTest Test, IReadOnlyList<string> ConstraintKeys, TaskCompletionSource<bool> StartSignal)>();
+
             lock (lockObject)
             {
                 // Release all constraint keys for this test
                 foreach (var key in constraintKeys)
                 {
                     lockedKeys.Remove(key);
                 }
-                
+
                 // Check waiting tests to see if any can now run
-                var tempQueue = new List<(AbstractExecutableTest Test, IReadOnlyList<string> ConstraintKeys, TaskCompletionSource<bool> StartSignal)>();
 
                 while (waitingTests.TryDequeue(out var waitingTest))
                 {
-                    // Check if all constraint keys are available for this waiting test
-                    var canStart = !waitingTest.ConstraintKeys.Any(key => lockedKeys.Contains(key));
-
+                    // Check if all constraint keys are available for this waiting test - manual loop avoids LINQ allocation
+                    var canStart = true;
+                    var waitingKeyCount = waitingTest.ConstraintKeys.Count;
+                    for (var i = 0; i < waitingKeyCount; i++)
+                    {
+                        if (lockedKeys.Contains(waitingTest.ConstraintKeys[i]))
+                        {
+                            canStart = false;
+                            break;
+                        }
+                    }
+
                     if (canStart)
                     {
                         // Lock the keys for this test
-                        foreach (var key in waitingTest.ConstraintKeys)
+                        for (var i = 0; i < waitingKeyCount; i++)
                         {
-                            lockedKeys.Add(key);
+                            lockedKeys.Add(waitingTest.ConstraintKeys[i]);
                         }
-                        
+
                         // Mark test to start after we exit the lock
                         testsToStart.Add(waitingTest);
                     }
                     else
                     {
                         // Still can't run, keep it in the queue
-                        tempQueue.Add(waitingTest);
+                        testsToRequeue.Add(waitingTest);
                     }
                 }
-                
+
                 // Re-add tests that still can't run
-                foreach (var waitingTestItem in tempQueue)
+                foreach (var waitingTestItem in testsToRequeue)
                 {
                     waitingTests.Enqueue(waitingTestItem);
                 }

TUnit.PerformanceBenchmarks/Tests/ConstraintKeys/ConstraintKeyStressTests.cs

@@ -0,0 +1,47 @@
+namespace TUnit.PerformanceBenchmarks.Tests.ConstraintKeys;
+
+/// <summary>
+/// Stress tests for ConstraintKeyScheduler with overlapping constraint keys.
+/// These tests create high contention scenarios to exercise lock contention paths.
+/// </summary>
+public class ConstraintKeyStressTests
+{
+    // Single constraint key tests - 50 instances each, must run serially within key group
+    [Test, Repeat(50), NotInParallel("KeyA")]
+    public async Task Test_SingleKey_A() => await Task.Delay(1);
+
+    [Test, Repeat(50), NotInParallel("KeyB")]
+    public async Task Test_SingleKey_B() => await Task.Delay(1);
+
+    [Test, Repeat(50), NotInParallel("KeyC")]
+    public async Task Test_SingleKey_C() => await Task.Delay(1);
+
+    [Test, Repeat(50), NotInParallel("KeyD")]
+    public async Task Test_SingleKey_D() => await Task.Delay(1);
+
+    [Test, Repeat(50), NotInParallel("KeyE")]
+    public async Task Test_SingleKey_E() => await Task.Delay(1);
+
+    // Overlapping two-key tests - creates dependency chains between key groups
+    [Test, Repeat(30), NotInParallel(["KeyA", "KeyB"])]
+    public async Task Test_OverlappingKeys_AB() => await Task.Delay(1);
+
+    [Test, Repeat(30), NotInParallel(["KeyB", "KeyC"])]
+    public async Task Test_OverlappingKeys_BC() => await Task.Delay(1);
+
+    [Test, Repeat(30), NotInParallel(["KeyC", "KeyD"])]
+    public async Task Test_OverlappingKeys_CD() => await Task.Delay(1);
+
+    [Test, Repeat(30), NotInParallel(["KeyD", "KeyE"])]
+    public async Task Test_OverlappingKeys_DE() => await Task.Delay(1);
+
+    [Test, Repeat(30), NotInParallel(["KeyE", "KeyA"])]
+    public async Task Test_OverlappingKeys_EA() => await Task.Delay(1);
+
+    // Triple-key tests - maximum contention scenarios
+    [Test, Repeat(20), NotInParallel(["KeyA", "KeyB", "KeyC"])]
+    public async Task Test_TripleKeys_ABC() => await Task.Delay(1);
+
+    [Test, Repeat(20), NotInParallel(["KeyC", "KeyD", "KeyE"])]
+    public async Task Test_TripleKeys_CDE() => await Task.Delay(1);
+}