[7.0] Make thread pool thread timeouts configurable

src/coreclr/inc/clrconfigvalues.h

@@ -542,6 +542,8 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("Thread
 #else // !TARGET_ARM64
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("ThreadPool_UnfairSemaphoreSpinLimit"), 0x46, "Maximum number of spins a thread pool worker thread performs before waiting for work")
 #endif // TARGET_ARM64
+RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_ThreadPool_ThreadTimeoutMs, W("ThreadPool_ThreadTimeoutMs"), (DWORD)-2, "The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and exiting. Set to -1 to disable the timeout. Applies to worker threads, completion port threads, and wait threads. Also see the ThreadPool_ThreadsToKeepAlive config value for relevant information.", CLRConfig::LookupOptions::ParseIntegerAsBase10)
+RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_ThreadPool_ThreadsToKeepAlive, W("ThreadPool_ThreadsToKeepAlive"), 0, "The number of worker or completion port threads to keep alive after they are created. Set to -1 to keep all created worker or completion port threads alive. When the ThreadPool_ThreadTimeoutMs config value is also set, for worker and completion port threads the timeout applies to threads in the respective pool that are in excess of the number configured for ThreadPool_ThreadsToKeepAlive.", CLRConfig::LookupOptions::ParseIntegerAsBase10)
 
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_Disable,                             W("HillClimbing_Disable"),                            0, "Disables hill climbing for thread adjustments in the thread pool");
 RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_WavePeriod,                          W("HillClimbing_WavePeriod"),                         4, "");

src/coreclr/vm/comthreadpool.cpp

@@ -192,6 +192,34 @@ FCIMPL4(INT32, ThreadPoolNative::GetNextConfigUInt32Value,
         case 19: if (TryGetConfig(CLRConfig::INTERNAL_HillClimbing_SampleIntervalHigh, false, W("System.Threading.ThreadPool.HillClimbing.SampleIntervalHigh"))) { return 20; } FALLTHROUGH;
         case 20: if (TryGetConfig(CLRConfig::INTERNAL_HillClimbing_GainExponent, false, W("System.Threading.ThreadPool.HillClimbing.GainExponent"))) { return 21; } FALLTHROUGH;
 
+        case 21:
+        {
+            int threadPoolThreadTimeoutMs = g_pConfig->ThreadPoolThreadTimeoutMs();
+            if (threadPoolThreadTimeoutMs >= -1)
+            {
+                *configValueRef = (UINT32)threadPoolThreadTimeoutMs;
+                *isBooleanRef = false;
+                *appContextConfigNameRef = W("System.Threading.ThreadPool.ThreadTimeoutMs");
+                return 22;
+            }
+
+            FALLTHROUGH;
+        }
+
+        case 22:
+        {
+            int threadPoolThreadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive();
+            if (threadPoolThreadsToKeepAlive >= -1)
+            {
+                *configValueRef = (UINT32)threadPoolThreadsToKeepAlive;
+                *isBooleanRef = false;
+                *appContextConfigNameRef = W("System.Threading.ThreadPool.ThreadsToKeepAlive");
+                return 23;
+            }
+
+            FALLTHROUGH;
+        }
+
         default:
             *configValueRef = 0;
             *isBooleanRef = false;

src/coreclr/vm/eeconfig.cpp

@@ -225,6 +225,9 @@ HRESULT EEConfig::Init()
     bDiagnosticSuspend = false;
 #endif
 
+    threadPoolThreadTimeoutMs = -2; // not configured
+    threadPoolThreadsToKeepAlive = 0;
+
 #if defined(FEATURE_TIERED_COMPILATION)
     fTieredCompilation = false;
     fTieredCompilation_QuickJit = false;
@@ -668,6 +671,19 @@ HRESULT EEConfig::sync()
 
 #endif //_DEBUG
 
+    threadPoolThreadTimeoutMs =
+        (int)Configuration::GetKnobDWORDValue(
+            W("System.Threading.ThreadPool.ThreadTimeoutMs"),
+            CLRConfig::EXTERNAL_ThreadPool_ThreadTimeoutMs);
+    threadPoolThreadsToKeepAlive =
+        (int)Configuration::GetKnobDWORDValue(
+            W("System.Threading.ThreadPool.ThreadsToKeepAlive"),
+            CLRConfig::EXTERNAL_ThreadPool_ThreadsToKeepAlive);
+    if (threadPoolThreadsToKeepAlive < -1)
+    {
+        threadPoolThreadsToKeepAlive = 0;
+    }
+
     m_fInteropValidatePinnedObjects = (CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_InteropValidatePinnedObjects) != 0);
     m_fInteropLogArguments = (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_InteropLogArguments) != 0);
 

src/coreclr/vm/eeconfig.h

@@ -469,6 +469,9 @@ class EEConfig
 
 #endif
 
+    int ThreadPoolThreadTimeoutMs() const { LIMITED_METHOD_CONTRACT; return threadPoolThreadTimeoutMs; }
+    int ThreadPoolThreadsToKeepAlive() const { LIMITED_METHOD_CONTRACT; return threadPoolThreadsToKeepAlive; }
+
 private: //----------------------------------------------------------------
 
     bool fInited;                   // have we synced to the registry at least once?
@@ -644,6 +647,9 @@ class EEConfig
     DWORD testThreadAbort;
 #endif
 
+    int threadPoolThreadTimeoutMs;
+    int threadPoolThreadsToKeepAlive;
+
 #if defined(FEATURE_TIERED_COMPILATION)
     bool fTieredCompilation;
     bool fTieredCompilation_QuickJit;

src/coreclr/vm/win32threadpool.cpp

@@ -125,6 +125,10 @@ SPTR_IMPL(WorkRequest,ThreadpoolMgr,WorkRequestTail);        // Head of work req
 unsigned int ThreadpoolMgr::LastCPThreadCreation=0;     //  last time a completion port thread was created
 unsigned int ThreadpoolMgr::NumberOfProcessors; // = NumberOfWorkerThreads - no. of blocked threads
 
+DWORD ThreadpoolMgr::WorkerThreadTimeoutMs = 20 * 1000;
+DWORD ThreadpoolMgr::IOCompletionThreadTimeoutMs = 15 * 1000;
+int ThreadpoolMgr::NumWorkerThreadsBeingKeptAlive = 0;
+int ThreadpoolMgr::NumIOCompletionThreadsBeingKeptAlive = 0;
 
 CrstStatic ThreadpoolMgr::WorkerCriticalSection;
 CLREvent * ThreadpoolMgr::RetiredCPWakeupEvent;       // wakeup event for completion port threads
@@ -344,6 +348,13 @@ BOOL ThreadpoolMgr::Initialize()
     NumberOfProcessors = GetCurrentProcessCpuCount();
     InitPlatformVariables();
 
+    int threadTimeoutMs = g_pConfig->ThreadPoolThreadTimeoutMs();
+    if (threadTimeoutMs >= -1)
+    {
+        WorkerThreadTimeoutMs = (DWORD)threadTimeoutMs;
+        IOCompletionThreadTimeoutMs = (DWORD)threadTimeoutMs;
+    }
+
     EX_TRY
     {
         if (!UsePortableThreadPool())
@@ -1747,6 +1758,9 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs)
     ThreadCounter::Counts counts, oldCounts, newCounts;
     bool foundWork = true, wasNotRecalled = true;
 
+    bool isThreadKeepAliveInitialized = false;
+    bool keepThreadAlive = false;
+
     counts = WorkerCounter.GetCleanCounts();
     if (ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context, ThreadPoolWorkerThreadStart))
         FireEtwThreadPoolWorkerThreadStart(counts.NumActive, counts.NumRetired, GetClrInstanceId());
@@ -1803,6 +1817,35 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs)
     GCX_PREEMP_NO_DTOR();
     _ASSERTE(pThread == NULL || !pThread->PreemptiveGCDisabled());
 
+    if (!isThreadKeepAliveInitialized && fThreadInit)
+    {
+        // Determine whether to keep this thread alive. Some threads may always be kept alive based on config.
+        isThreadKeepAliveInitialized = true;
+        int threadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive();
+        if (threadsToKeepAlive != 0)
+        {
+            if (threadsToKeepAlive < 0)
+            {
+                keepThreadAlive = true;
+            }
+            else
+            {
+                int count = VolatileLoadWithoutBarrier(&NumWorkerThreadsBeingKeptAlive);
+                while (count < threadsToKeepAlive)
+                {
+                    int countBeforeUpdate = InterlockedCompareExchangeT(&NumWorkerThreadsBeingKeptAlive, count + 1, count);
+                    if (countBeforeUpdate == count)
+                    {
+                        keepThreadAlive = true;
+                        break;
+                    }
+
+                    count = countBeforeUpdate;
+                }
+            }
+        }
+    }
+
     // make sure there's really work.  If not, go back to sleep
 
     // counts volatile read paired with CompareExchangeCounts loop set
@@ -1889,7 +1932,7 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs)
     while (true)
     {
 RetryRetire:
-        if (RetiredWorkerSemaphore->Wait(WorkerTimeout))
+        if (RetiredWorkerSemaphore->Wait(keepThreadAlive ? INFINITE : WorkerThreadTimeoutMs))
         {
             foundWork = true;
 
@@ -1966,7 +2009,7 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs)
         FireEtwThreadPoolWorkerThreadWait(counts.NumActive, counts.NumRetired, GetClrInstanceId());
 
 RetryWaitForWork:
-    if (WorkerSemaphore->Wait(WorkerTimeout, WorkerThreadSpinLimit, NumberOfProcessors))
+    if (WorkerSemaphore->Wait(keepThreadAlive ? INFINITE : WorkerThreadTimeoutMs, WorkerThreadSpinLimit, NumberOfProcessors))
     {
         foundWork = true;
         goto Work;
@@ -3080,13 +3123,13 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs)
     PIOCompletionContext context;
     BOOL fIsCompletionContext;
 
-    const DWORD CP_THREAD_WAIT = 15000; /* milliseconds */
-
     _ASSERTE(GlobalCompletionPort != NULL);
 
     BOOL fThreadInit = FALSE;
     Thread *pThread = NULL;
 
+    bool isThreadKeepAliveInitialized = false;
+    bool keepThreadAlive = false;
     DWORD cpThreadWait = 0;
 
     if (g_fEEStarted) {
@@ -3123,7 +3166,7 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs)
     ThreadCounter::Counts oldCounts;
     ThreadCounter::Counts newCounts;
 
-    cpThreadWait = CP_THREAD_WAIT;
+    cpThreadWait = IOCompletionThreadTimeoutMs;
     for (;; )
     {
 Top:
@@ -3151,6 +3194,36 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs)
 
         GCX_PREEMP_NO_DTOR();
 
+        if (!isThreadKeepAliveInitialized && fThreadInit)
+        {
+            // Determine whether to keep this thread alive. Some threads may always be kept alive based on config.
+            isThreadKeepAliveInitialized = true;
+            int threadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive();
+            if (threadsToKeepAlive != 0)
+            {
+                if (threadsToKeepAlive < 0)
+                {
+                    keepThreadAlive = true;
+                }
+                else
+                {
+                    int count = VolatileLoadWithoutBarrier(&NumIOCompletionThreadsBeingKeptAlive);
+                    while (count < threadsToKeepAlive)
+                    {
+                        int countBeforeUpdate =
+                            InterlockedCompareExchangeT(&NumIOCompletionThreadsBeingKeptAlive, count + 1, count);
+                        if (countBeforeUpdate == count)
+                        {
+                            keepThreadAlive = true;
+                            break;
+                        }
+
+                        count = countBeforeUpdate;
+                    }
+                }
+            }
+        }
+
         //
         // We're about to wait on the IOCP; mark ourselves as no longer "working."
         //
@@ -3165,7 +3238,7 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs)
             // one thread listening for completions.  So there's no point in having a timeout; it will
             // only use power unnecessarily.
             //
-            cpThreadWait = (newCounts.NumActive == 1) ? INFINITE : CP_THREAD_WAIT;
+            cpThreadWait = (newCounts.NumActive == 1 || keepThreadAlive) ? INFINITE : IOCompletionThreadTimeoutMs;
 
             if (oldCounts == CPThreadCounter.CompareExchangeCounts(newCounts, oldCounts))
                 break;

src/coreclr/vm/win32threadpool.h

@@ -971,6 +971,11 @@ class ThreadpoolMgr
     static unsigned int LastCPThreadCreation;		// last time a completion port thread was created
     static unsigned int NumberOfProcessors;             // = NumberOfWorkerThreads - no. of blocked threads
 
+    static DWORD WorkerThreadTimeoutMs;
+    static DWORD IOCompletionThreadTimeoutMs;
+    static int NumWorkerThreadsBeingKeptAlive;
+    static int NumIOCompletionThreadsBeingKeptAlive;
+
     static BOOL IsApcPendingOnWaitThread;               // Indicates if an APC is pending on the wait thread
 
     // This needs to be non-hosted, because worker threads can run prior to EE startup.
@@ -980,8 +985,6 @@ class ThreadpoolMgr
     static CrstStatic WorkerCriticalSection;
 
 private:
-    static const DWORD WorkerTimeout = 20 * 1000;
-
     DECLSPEC_ALIGN(MAX_CACHE_LINE_SIZE) SVAL_DECL(ThreadCounter,WorkerCounter);
 
     //

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs

@@ -7,6 +7,8 @@ namespace System.Threading
 {
     internal sealed partial class PortableThreadPool
     {
+        private int _numThreadsBeingKeptAlive;
+
         /// <summary>
         /// The worker thread infastructure for the CLR thread pool.
         /// </summary>
@@ -28,6 +30,22 @@ private static partial class WorkerThread
             // preexisting threads from running out of memory when using new stack space in low-memory situations.
             public const int EstimatedAdditionalStackUsagePerThreadBytes = 64 << 10; // 64 KB
 
+            private static readonly short ThreadsToKeepAlive = DetermineThreadsToKeepAlive();
+
+            private static short DetermineThreadsToKeepAlive()
+            {
+                const short DefaultThreadsToKeepAlive = 0;
+
+                // The number of worker threads to keep alive after they are created. Set to -1 to keep all created worker
+                // threads alive. When the ThreadTimeoutMs config value is also set, for worker threads the timeout applies to
+                // worker threads that are in excess of the number configured for ThreadsToKeepAlive.
+                short threadsToKeepAlive =
+                    AppContextConfigHelper.GetInt16Config(
+                        "System.Threading.ThreadPool.ThreadsToKeepAlive",
+                        DefaultThreadsToKeepAlive);
+                return threadsToKeepAlive >= -1 ? threadsToKeepAlive : DefaultThreadsToKeepAlive;
+            }
+
             /// <summary>
             /// Semaphore for controlling how many threads are currently working.
             /// </summary>
@@ -65,10 +83,36 @@ private static void WorkerThreadStart()
                 LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock;
                 LowLevelLifoSemaphore semaphore = s_semaphore;
 
+                // Determine the idle timeout to use for this thread. Some threads may always be kept alive based on config.
+                int timeoutMs = ThreadPoolThreadTimeoutMs;
+                if (ThreadsToKeepAlive != 0)
+                {
+                    if (ThreadsToKeepAlive < 0)
+                    {
+                        timeoutMs = Timeout.Infinite;
+                    }
+                    else
+                    {
+                        int count = threadPoolInstance._numThreadsBeingKeptAlive;
+                        while (count < ThreadsToKeepAlive)
+                        {
+                            int countBeforeUpdate =
+                                Interlocked.CompareExchange(ref threadPoolInstance._numThreadsBeingKeptAlive, count + 1, count);
+                            if (countBeforeUpdate == count)
+                            {
+                                timeoutMs = Timeout.Infinite;
+                                break;
+                            }
+
+                            count = countBeforeUpdate;
+                        }
+                    }
+                }
+
                 while (true)
                 {
                     bool spinWait = true;
-                    while (semaphore.Wait(ThreadPoolThreadTimeoutMs, spinWait))
+                    while (semaphore.Wait(timeoutMs, spinWait))
                     {
                         bool alreadyRemovedWorkingWorker = false;
                         while (TakeActiveRequest(threadPoolInstance))

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs

@@ -13,7 +13,6 @@ namespace System.Threading
     /// </summary>
     internal sealed partial class PortableThreadPool
     {
-        private const int ThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests.
         private const int SmallStackSizeBytes = 256 * 1024;
 
         private const short MaxPossibleThreadCount = short.MaxValue;
@@ -34,6 +33,22 @@ internal sealed partial class PortableThreadPool
         private static readonly short ForcedMaxWorkerThreads =
             AppContextConfigHelper.GetInt16Config("System.Threading.ThreadPool.MaxThreads", 0, false);
 
+        private static readonly int ThreadPoolThreadTimeoutMs = DetermineThreadPoolThreadTimeoutMs();
+
+        private static int DetermineThreadPoolThreadTimeoutMs()
+        {
+            const int DefaultThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests.
+
+            // The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and
+            // exiting. Set to -1 to disable the timeout. Applies to worker threads and wait threads. Also see the
+            // ThreadsToKeepAlive config value for relevant information.
+            int timeoutMs =
+                AppContextConfigHelper.GetInt32Config(
+                    "System.Threading.ThreadPool.ThreadTimeoutMs",
+                    DefaultThreadPoolThreadTimeoutMs);
+            return timeoutMs >= -1 ? timeoutMs : DefaultThreadPoolThreadTimeoutMs;
+        }
+
         [ThreadStatic]
         private static object? t_completionCountObject;