Expose and test APIs for some threading metrics (CoreFX) (dotnet/core…

…fx#37401)

* Expose and test APIs for some threading metrics (CoreFX)

- API review: https://github.com/dotnet/corefx/issues/35500
- Depends on dotnet/coreclr#22754, dotnet/corert#7066

* Separate and expose pending local vs global work item count

* Remove local/global variants of PendingWorkItemCount

* Remove unrelated test

* Add test for a fix to ThreadLocal.Values property throwing NullReferenceException when disposed

Fix is in dotnet/corert#7066

* Fix build

* Fix test

* Add API compat baselines for uapaot

* Fix test

* Use RemoteExecutor for MetricsTest

* Address feedback


Commit migrated from dotnet/corefx@34fe566

src/libraries/Common/tests/System/Threading/ThreadTestHelpers.cs

@@ -123,13 +123,29 @@ public static void WaitForConditionWithoutBlocking(Func<bool> condition)
             WaitForConditionWithCustomDelay(condition, () => Thread.Yield());
         }
 
+        public static void WaitForConditionWithoutRelinquishingTimeSlice(Func<bool> condition)
+        {
+            WaitForConditionWithCustomDelay(condition, () => Thread.SpinWait(1));
+        }
+
         public static void WaitForConditionWithCustomDelay(Func<bool> condition, Action delay)
         {
-            var startTime = DateTime.Now;
-            while (!condition())
+            if (condition())
+            {
+                return;
+            }
+
+            int startTimeMs = Environment.TickCount;
+            while (true)
             {
-                Assert.True((DateTime.Now - startTime).TotalMilliseconds < UnexpectedTimeoutMilliseconds);
                 delay();
+
+                if (condition())
+                {
+                    return;
+                }
+
+                Assert.InRange(Environment.TickCount - startTimeMs, 0, UnexpectedTimeoutMilliseconds);
             }
         }
 

src/libraries/System.Threading.Overlapped/tests/Configurations.props

@@ -1,6 +1,7 @@
 <Project DefaultTargets="Build">
   <PropertyGroup>
     <BuildConfigurations>
+      netcoreapp;
       netstandard;
       uap-Windows_NT;
     </BuildConfigurations>

src/libraries/System.Threading.Overlapped/tests/System.Threading.Overlapped.Tests.csproj

@@ -2,13 +2,15 @@
   <PropertyGroup>
     <ProjectGuid>{861A3318-35AD-46ac-8257-8D5D2479BAD9}</ProjectGuid>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <Configurations>netstandard-Debug;netstandard-Release;uap-Windows_NT-Debug;uap-Windows_NT-Release</Configurations>
+    <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
+    <Configurations>netcoreapp-Debug;netcoreapp-Release;netstandard-Debug;netstandard-Release;uap-Windows_NT-Debug;uap-Windows_NT-Release</Configurations>
     <TestRuntime>true</TestRuntime>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="DllImport.cs" />
     <Compile Include="ThreadPoolBoundHandle_PreAllocatedOverlappedTests.cs" />
     <Compile Include="ThreadPoolBoundHandle_IntegrationTests.cs" />
+    <Compile Include="ThreadPoolBoundHandle_IntegrationTests.netcoreapp.cs" Condition="'$(TargetGroup)' != 'netstandard'" />
     <Compile Include="ThreadPoolBoundHandle_Helpers.cs" />
     <Compile Include="AsyncResult.cs" />
     <Compile Include="Win32Handle.cs" />
@@ -21,4 +23,9 @@
     <Compile Include="ThreadPoolBoundHandle_BindHandleTests.cs" />
     <Compile Include="OverlappedTests.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <Compile Include="$(CommonTestPath)\System\Threading\ThreadTestHelpers.cs">
+      <Link>CommonTest\System\Threading\ThreadTestHelpers.cs</Link>
+    </Compile>
+  </ItemGroup>
 </Project>

src/libraries/System.Threading.Overlapped/tests/ThreadPoolBoundHandle_IntegrationTests.netcoreapp.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Threading;
+using System.Threading.Tests;
+using Xunit;
+
+public partial class ThreadPoolBoundHandleTests
+{
+    [Fact]
+    [PlatformSpecific(TestPlatforms.Windows)] // ThreadPoolBoundHandle.BindHandle is not supported on Unix
+    public unsafe void MultipleOperationsOverSingleHandle_CompletedWorkItemCountTest()
+    {
+        long initialCompletedWorkItemCount = ThreadPool.CompletedWorkItemCount;
+        MultipleOperationsOverMultipleHandles();
+        ThreadTestHelpers.WaitForCondition(() => ThreadPool.CompletedWorkItemCount - initialCompletedWorkItemCount >= 2);
+    }
+}

src/libraries/System.Threading.Thread/tests/System.Threading.Thread.Tests.csproj

@@ -17,7 +17,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="$(CommonTestPath)\System\Threading\ThreadTestHelpers.cs">
-      <Link>CommonTest\System\Threading\ThreadPoolHelpers.cs</Link>
+      <Link>CommonTest\System\Threading\ThreadTestHelpers.cs</Link>
     </Compile>
     <ProjectReference Include="STAMain\STAMain.csproj">
       <Name>STAMain</Name>

src/libraries/System.Threading.Thread/tests/ThreadTests.netcoreapp.cs

@@ -8,6 +8,7 @@ namespace System.Threading.Threads.Tests
 {
     public static partial class ThreadTests
     {
+        [Fact]
         public static void GetCurrentProcessorId()
         {
             Assert.True(Thread.GetCurrentProcessorId() >= 0);

src/libraries/System.Threading.ThreadPool/ref/System.Threading.ThreadPool.cs

@@ -21,9 +21,11 @@ public static partial class ThreadPool
         [System.ObsoleteAttribute("ThreadPool.BindHandle(IntPtr) has been deprecated.  Please use ThreadPool.BindHandle(SafeHandle) instead.", false)]
         public static bool BindHandle(System.IntPtr osHandle) { throw null; }
         public static bool BindHandle(System.Runtime.InteropServices.SafeHandle osHandle) { throw null; }
+        public static long CompletedWorkItemCount { get { throw null; } }
         public static void GetAvailableThreads(out int workerThreads, out int completionPortThreads) { throw null; }
         public static void GetMaxThreads(out int workerThreads, out int completionPortThreads) { throw null; }
         public static void GetMinThreads(out int workerThreads, out int completionPortThreads) { throw null; }
+        public static long PendingWorkItemCount { get { throw null; } }
         public static bool QueueUserWorkItem(System.Threading.WaitCallback callBack) { throw null; }
         public static bool QueueUserWorkItem(System.Threading.WaitCallback callBack, object state) { throw null; }
         public static bool QueueUserWorkItem<TState>(System.Action<TState> callBack, TState state, bool preferLocal) { throw null; }
@@ -34,6 +36,7 @@ public static partial class ThreadPool
         public static System.Threading.RegisteredWaitHandle RegisterWaitForSingleObject(System.Threading.WaitHandle waitObject, System.Threading.WaitOrTimerCallback callBack, object state, uint millisecondsTimeOutInterval, bool executeOnlyOnce) { throw null; }
         public static bool SetMaxThreads(int workerThreads, int completionPortThreads) { throw null; }
         public static bool SetMinThreads(int workerThreads, int completionPortThreads) { throw null; }
+        public static int ThreadCount { get { throw null; } }
         [System.CLSCompliantAttribute(false)]
         public unsafe static bool UnsafeQueueNativeOverlapped(System.Threading.NativeOverlapped* overlapped) { throw null; }
         public static bool UnsafeQueueUserWorkItem(System.Threading.IThreadPoolWorkItem callBack, bool preferLocal) { throw null; }

src/libraries/System.Threading.ThreadPool/src/ApiCompatBaseline.uapaot.txt

@@ -0,0 +1,4 @@
+Compat issues with assembly System.Threading.ThreadPool:
+MembersMustExist : Member 'System.Threading.ThreadPool.CompletedWorkItemCount.get()' does not exist in the implementation but it does exist in the contract.
+MembersMustExist : Member 'System.Threading.ThreadPool.PendingWorkItemCount.get()' does not exist in the implementation but it does exist in the contract.
+MembersMustExist : Member 'System.Threading.ThreadPool.ThreadCount.get()' does not exist in the implementation but it does exist in the contract.

src/libraries/System.Threading.ThreadPool/tests/Configurations.props

@@ -3,6 +3,7 @@
     <BuildConfigurations>
       netcoreapp;
       netstandard;
+      uap;
     </BuildConfigurations>
   </PropertyGroup>
 </Project>

src/libraries/System.Threading.ThreadPool/tests/System.Threading.ThreadPool.Tests.csproj

@@ -2,12 +2,12 @@
   <PropertyGroup>
     <ProjectGuid>{403AD1B8-6F95-4A2E-92A2-727606ABD866}</ProjectGuid>
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
-    <Configurations>netcoreapp-Debug;netcoreapp-Release;netstandard-Debug;netstandard-Release</Configurations>
+    <Configurations>netcoreapp-Debug;netcoreapp-Release;netstandard-Debug;netstandard-Release;uap-Debug;uap-Release</Configurations>
     <TestRuntime>true</TestRuntime>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="ThreadPoolTests.cs" />
-    <Compile Include="ThreadPoolTests.netcoreapp.cs" Condition="'$(TargetGroup)' == 'netcoreapp'" />
+    <Compile Include="ThreadPoolTests.netcoreapp.cs" Condition="'$(TargetGroup)' != 'netstandard'" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="$(CommonTestPath)\System\Threading\ThreadTestHelpers.cs">

src/libraries/System.Threading.ThreadPool/tests/ThreadPoolTests.netcoreapp.cs

@@ -5,6 +5,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using System.Threading.Tests;
+using Microsoft.DotNet.RemoteExecutor;
 using Xunit;
 
 namespace System.Threading.ThreadPools.Tests
@@ -188,5 +190,145 @@ private sealed class InvalidWorkItemAndTask : Task, IThreadPoolWorkItem
             public InvalidWorkItemAndTask(Action action) : base(action) { }
             public void Execute() { }
         }
+
+        [ConditionalFact(nameof(HasAtLeastThreeProcessors))]
+        public void MetricsTest()
+        {
+            RemoteExecutor.Invoke(() =>
+            {
+                int processorCount = Environment.ProcessorCount;
+                if (processorCount <= 2)
+                {
+                    return;
+                }
+
+                bool waitForWorkStart = false;
+                var workStarted = new AutoResetEvent(false);
+                var localWorkScheduled = new AutoResetEvent(false);
+                int completeWork = 0;
+                int queuedWorkCount = 0;
+                var allWorkCompleted = new ManualResetEvent(false);
+                Exception backgroundEx = null;
+                Action work = () =>
+                {
+                    if (waitForWorkStart)
+                    {
+                        workStarted.Set();
+                    }
+                    try
+                    {
+                        // Blocking can affect thread pool thread injection heuristics, so don't block, pretend like a
+                        // long-running CPU-bound work item
+                        ThreadTestHelpers.WaitForConditionWithoutRelinquishingTimeSlice(
+                                () => Interlocked.CompareExchange(ref completeWork, 0, 0) != 0);
+                    }
+                    catch (Exception ex)
+                    {
+                        Interlocked.CompareExchange(ref backgroundEx, ex, null);
+                    }
+                    finally
+                    {
+                        if (Interlocked.Decrement(ref queuedWorkCount) == 0)
+                        {
+                            allWorkCompleted.Set();
+                        }
+                    }
+                };
+                WaitCallback threadPoolGlobalWork = data => work();
+                Action<object> threadPoolLocalWork = data => work();
+                WaitCallback scheduleThreadPoolLocalWork = data =>
+                {
+                    try
+                    {
+                        int n = (int)data;
+                        for (int i = 0; i < n; ++i)
+                        {
+                            ThreadPool.QueueUserWorkItem(threadPoolLocalWork, null, preferLocal: true);
+                            if (waitForWorkStart)
+                            {
+                                workStarted.CheckedWait();
+                            }
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Interlocked.CompareExchange(ref backgroundEx, ex, null);
+                    }
+                    finally
+                    {
+                        localWorkScheduled.Set();
+                    }
+                };
+
+                var signaledEvent = new ManualResetEvent(true);
+                var timers = new List<Timer>();
+                int totalWorkCountToQueue = 0;
+                Action scheduleWork = () =>
+                {
+                    Assert.True(queuedWorkCount < totalWorkCountToQueue);
+
+                    int workCount = (totalWorkCountToQueue - queuedWorkCount) / 2;
+                    if (workCount > 0)
+                    {
+                        queuedWorkCount += workCount;
+                        ThreadPool.QueueUserWorkItem(scheduleThreadPoolLocalWork, workCount);
+                        localWorkScheduled.CheckedWait();
+                    }
+
+                    for (; queuedWorkCount < totalWorkCountToQueue; ++queuedWorkCount)
+                    {
+                        ThreadPool.QueueUserWorkItem(threadPoolGlobalWork);
+                        if (waitForWorkStart)
+                        {
+                            workStarted.CheckedWait();
+                        }
+                    }
+                };
+
+                Interlocked.MemoryBarrierProcessWide(); // get a reasonably accurate value for the following
+                long initialCompletedWorkItemCount = ThreadPool.CompletedWorkItemCount;
+
+                try
+                {
+                    // Schedule some simultaneous work that would all be scheduled and verify the thread count
+                    totalWorkCountToQueue = processorCount - 2;
+                    Assert.True(totalWorkCountToQueue >= 1);
+                    waitForWorkStart = true;
+                    scheduleWork();
+                    Assert.True(ThreadPool.ThreadCount >= totalWorkCountToQueue);
+
+                    int runningWorkItemCount = queuedWorkCount;
+
+                    // Schedule more work that would not all be scheduled and roughly verify the pending work item count
+                    totalWorkCountToQueue = processorCount * 64;
+                    waitForWorkStart = false;
+                    scheduleWork();
+                    int minExpectedPendingWorkCount = Math.Max(1, queuedWorkCount - runningWorkItemCount * 8);
+                    ThreadTestHelpers.WaitForCondition(() => ThreadPool.PendingWorkItemCount >= minExpectedPendingWorkCount);
+                }
+                finally
+                {
+                    // Complete the work
+                    Interlocked.Exchange(ref completeWork, 1);
+                }
+
+                // Wait for work items to exit, for counting
+                allWorkCompleted.CheckedWait();
+                backgroundEx = Interlocked.CompareExchange(ref backgroundEx, null, null);
+                if (backgroundEx != null)
+                {
+                    throw new AggregateException(backgroundEx);
+                }
+
+                // Verify the completed work item count
+                ThreadTestHelpers.WaitForCondition(() =>
+                {
+                    Interlocked.MemoryBarrierProcessWide(); // get a reasonably accurate value for the following
+                    return ThreadPool.CompletedWorkItemCount - initialCompletedWorkItemCount >= totalWorkCountToQueue;
+                });
+            }).Dispose();
+        }
+
+        public static bool HasAtLeastThreeProcessors => Environment.ProcessorCount >= 3;
     }
 }

src/libraries/System.Threading.Timer/tests/Configurations.props

@@ -3,6 +3,7 @@
     <BuildConfigurations>
       netcoreapp;
       netstandard;
+      uap;
     </BuildConfigurations>
   </PropertyGroup>
 </Project>

src/libraries/System.Threading.Timer/tests/System.Threading.Timer.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <ProjectGuid>{ac20a28f-fda8-45e8-8728-058ead16e44c}</ProjectGuid>
-    <Configurations>netcoreapp-Debug;netcoreapp-Release;netstandard-Debug;netstandard-Release</Configurations>
+    <Configurations>netcoreapp-Debug;netcoreapp-Release;netstandard-Debug;netstandard-Release;uap-Debug;uap-Release</Configurations>
     <TestRuntime>true</TestRuntime>
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
   </PropertyGroup>
@@ -10,12 +10,7 @@
     <Compile Include="TimerChangeTests.cs" />
     <Compile Include="TimerFiringTests.cs" />
   </ItemGroup>
-  <ItemGroup Condition="'$(TargetGroup)' == 'netcoreapp'">
+  <ItemGroup Condition="'$(TargetGroup)' != 'netstandard'">
     <Compile Include="TimerDisposeTests.cs" />
   </ItemGroup>
-  <ItemGroup>
-    <Compile Include="$(CommonTestPath)\System\Threading\ThreadTestHelpers.cs">
-      <Link>CommonTest\System\Threading\ThreadTestHelpers.cs</Link>
-    </Compile>
-  </ItemGroup>
 </Project>