Enable startup events over EventPipe (#36720)

* Add PauseOnStart inside ceemain 
* Add DOTNET_ env vars for opting out of pause on start
* Add Resume command to Diagnostics IPC

src/coreclr/src/inc/clrconfigvalues.h

@@ -710,6 +710,7 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_EventPipeProcNumbers, W("EventPipeProcNumbers"
 // Diagnostics Server
 //
 RETAIL_CONFIG_STRING_INFO_EX(EXTERNAL_DOTNET_DiagnosticsMonitorAddress, W("DOTNET_DiagnosticsMonitorAddress"), "NamedPipe path without '\\\\.\\pipe\\' on Windows; Full path of Unix Domain Socket on Linux/Unix.  Used for Diagnostics Monitoring Agents.", CLRConfig::DontPrependCOMPlus_);
+RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_DOTNET_DiagnosticsMonitorPauseOnStart, W("DOTNET_DiagnosticsMonitorPauseOnStart"), 1, "If DOTNET_DiagnosticsMonitorAddress is set, this will cause the runtime to pause during startup.  Resume using the Diagnostics IPC ResumeStartup command.", CLRConfig::DontPrependCOMPlus_);
 
 //
 // LTTng

src/coreclr/src/inc/eventtracebase.h

@@ -204,8 +204,7 @@ struct ProfilingScanContext;
 
 #define ETWOnStartup(StartEventName, EndEventName) \
     ETWTraceStartup trace##StartEventName##(Microsoft_Windows_DotNETRuntimePrivateHandle, &StartEventName, &StartupId, &EndEventName, &StartupId);
-#define ETWFireEvent(EventName) \
-    ETWTraceStartup::StartupTraceEvent(Microsoft_Windows_DotNETRuntimePrivateHandle, &EventName, &StartupId);
+#define ETWFireEvent(EventName) FireEtw##EventName(GetClrInstanceId())
 
 #ifndef FEATURE_REDHAWK
 // Headers

src/coreclr/src/vm/ceemain.cpp

@@ -674,13 +674,17 @@ void EEStartupHelper()
 #ifdef FEATURE_PERFTRACING
         // Initialize the event pipe.
         EventPipe::Initialize();
-
 #endif // FEATURE_PERFTRACING
 
 #ifdef TARGET_UNIX
         PAL_SetShutdownCallback(EESocketCleanupHelper);
 #endif // TARGET_UNIX
 
+#ifdef FEATURE_PERFTRACING
+        DiagnosticServer::Initialize();
+        DiagnosticServer::PauseForDiagnosticsMonitor();
+#endif // FEATURE_PERFTRACING
+
 #ifdef FEATURE_GDBJIT
         // Initialize gdbjit
         NotifyGdb::Initialize();
@@ -933,12 +937,12 @@ void EEStartupHelper()
         hr = g_pGCHeap->Initialize();
         IfFailGo(hr);
 
-#ifdef FEATURE_EVENT_TRACE
+#ifdef FEATURE_PERFTRACING
         // Finish setting up rest of EventPipe - specifically enable SampleProfiler if it was requested at startup.
         // SampleProfiler needs to cooperate with the GC which hasn't fully finished setting up in the first part of the
         // EventPipe initialization, so this is done after the GC has been fully initialized.
         EventPipe::FinishInitialize();
-#endif
+#endif // FEATURE_PERFTRACING
 
         // This isn't done as part of InitializeGarbageCollector() above because thread
         // creation requires AppDomains to have been set up.
@@ -1007,11 +1011,6 @@ void EEStartupHelper()
 #endif // CROSSGEN_COMPILE
 
         g_fEEStarted = TRUE;
-#ifndef CROSSGEN_COMPILE
-#ifdef FEATURE_PERFTRACING
-        DiagnosticServer::Initialize();
-#endif
-#endif
         g_EEStartupStatus = S_OK;
         hr = S_OK;
         STRESS_LOG0(LF_STARTUP, LL_ALWAYS, "===================EEStartup Completed===================");

src/coreclr/src/vm/diagnosticserver.cpp

@@ -21,6 +21,7 @@
 #ifdef FEATURE_PERFTRACING
 
 Volatile<bool> DiagnosticServer::s_shuttingDown(false);
+CLREventStatic *DiagnosticServer::s_ResumeRuntimeStartupEvent = nullptr;
 
 DWORD WINAPI DiagnosticServer::DiagnosticsServerThread(LPVOID)
 {
@@ -71,6 +72,10 @@ DWORD WINAPI DiagnosticServer::DiagnosticsServerThread(LPVOID)
 
             switch ((DiagnosticsIpc::DiagnosticServerCommandSet)message.GetHeader().CommandSet)
             {
+            case DiagnosticsIpc::DiagnosticServerCommandSet::Server:
+                DiagnosticServerProtocolHelper::HandleIpcMessage(message, pStream);
+                break;
+
             case DiagnosticsIpc::DiagnosticServerCommandSet::EventPipe:
                 EventPipeProtocolHelper::HandleIpcMessage(message, pStream);
                 break;
@@ -81,7 +86,7 @@ DWORD WINAPI DiagnosticServer::DiagnosticsServerThread(LPVOID)
 
 #ifdef FEATURE_PROFAPI_ATTACH_DETACH
             case DiagnosticsIpc::DiagnosticServerCommandSet::Profiler:
-                ProfilerDiagnosticProtocolHelper::AttachProfiler(message, pStream);
+                ProfilerDiagnosticProtocolHelper::HandleIpcMessage(message, pStream);
                 break;
 #endif // FEATURE_PROFAPI_ATTACH_DETACH
 
@@ -137,6 +142,10 @@ bool DiagnosticServer::Initialize()
         int nCharactersWritten = 0;
         if (wAddress != nullptr)
         {
+            // By default, opts in to Pause on Start
+            s_ResumeRuntimeStartupEvent = new CLREventStatic();
+            s_ResumeRuntimeStartupEvent->CreateManualEvent(false);
+
             nCharactersWritten = WideCharToMultiByte(CP_UTF8, 0, wAddress, -1, NULL, 0, NULL, NULL);
             if (nCharactersWritten != 0)
             {
@@ -238,4 +247,89 @@ bool DiagnosticServer::Shutdown()
     return fSuccess;
 }
 
+// This method will block runtime bring-up IFF DOTNET_DiagnosticsMonitorAddress != nullptr and DOTNET_DiagnosticsMonitorPauseOnStart!=0 (it's default state)
+// The s_ResumeRuntimeStartupEvent event will be signaled when the Diagnostics Monitor uses the ResumeRuntime Diagnostics IPC Command
+void DiagnosticServer::PauseForDiagnosticsMonitor()
+{
+    CONTRACTL
+    {
+      THROWS;
+      GC_NOTRIGGER;
+      MODE_PREEMPTIVE;
+    }
+    CONTRACTL_END;
+
+    CLRConfigStringHolder pDotnetDiagnosticsMonitorAddress = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_DOTNET_DiagnosticsMonitorAddress);
+    if (pDotnetDiagnosticsMonitorAddress != nullptr)
+    {
+        DWORD dwDotnetDiagnosticsMonitorPauseOnStart = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_DOTNET_DiagnosticsMonitorPauseOnStart);
+        if (dwDotnetDiagnosticsMonitorPauseOnStart != 0)
+        {
+            _ASSERTE(s_ResumeRuntimeStartupEvent != nullptr && s_ResumeRuntimeStartupEvent->IsValid());
+            wprintf(W("The runtime has been configured to pause during startup and is awaiting a Diagnostics IPC ResumeStartup command from a server at '%s'.\n"), (LPWSTR)pDotnetDiagnosticsMonitorAddress);
+            fflush(stdout);
+            STRESS_LOG0(LF_DIAGNOSTICS_PORT, LL_ALWAYS, "The runtime has been configured to pause during startup and is awaiting a Diagnostics IPC ResumeStartup command.");
+            const DWORD dwFiveSecondWait = s_ResumeRuntimeStartupEvent->Wait(5000, false);
+            if (dwFiveSecondWait == WAIT_TIMEOUT)
+            {
+                STRESS_LOG0(LF_DIAGNOSTICS_PORT, LL_ALWAYS, "The runtime has been configured to pause during startup and is awaiting a Diagnostics IPC ResumeStartup command and has waitied 5 seconds.");
+                const DWORD dwWait = s_ResumeRuntimeStartupEvent->Wait(INFINITE, false);
+            }
+        }
+    }
+    // allow wait failures to fall through and the runtime to continue coming up
+}
+
+void DiagnosticServer::ResumeRuntimeStartup()
+{
+    LIMITED_METHOD_CONTRACT;
+    if (s_ResumeRuntimeStartupEvent != nullptr && s_ResumeRuntimeStartupEvent->IsValid())
+        s_ResumeRuntimeStartupEvent->Set();
+}
+
+void DiagnosticServerProtocolHelper::HandleIpcMessage(DiagnosticsIpc::IpcMessage& message, IpcStream* pStream)
+{
+    CONTRACTL
+    {
+        THROWS;
+        GC_TRIGGERS;
+        MODE_ANY;
+        PRECONDITION(pStream != nullptr);
+    }
+    CONTRACTL_END;
+
+    switch ((DiagnosticsIpc::DiagnosticServerCommandId)message.GetHeader().CommandId)
+    {
+    case DiagnosticsIpc::DiagnosticServerCommandId::ResumeRuntime:
+        DiagnosticServerProtocolHelper::ResumeRuntimeStartup(message, pStream);
+        break;
+
+    default:
+        STRESS_LOG1(LF_DIAGNOSTICS_PORT, LL_WARNING, "Received unknown request type (%d)\n", message.GetHeader().CommandSet);
+        DiagnosticsIpc::IpcMessage::SendErrorMessage(pStream, CORDIAGIPC_E_UNKNOWN_COMMAND);
+        delete pStream;
+        break;
+    }
+}
+
+void DiagnosticServerProtocolHelper::ResumeRuntimeStartup(DiagnosticsIpc::IpcMessage& message, IpcStream *pStream)
+{
+    CONTRACTL
+    {
+        THROWS;
+        GC_TRIGGERS;
+        MODE_PREEMPTIVE;
+        PRECONDITION(pStream != nullptr);
+    }
+    CONTRACTL_END;
+
+    // no payload
+    DiagnosticServer::ResumeRuntimeStartup();
+    HRESULT res = S_OK;
+
+    DiagnosticsIpc::IpcMessage successResponse;
+    if (successResponse.Initialize(DiagnosticsIpc::GenericSuccessHeader, res))
+        successResponse.Send(pStream);
+}
+
 #endif // FEATURE_PERFTRACING

src/coreclr/src/vm/diagnosticserver.h

@@ -8,45 +8,41 @@
 #ifdef FEATURE_PERFTRACING // This macro should change to something more generic than performance.
 
 #include "diagnosticsipc.h"
-
-enum class DiagnosticMessageType : uint32_t
-{
-    ///////////////////////////////////////////////////////////////////////////
-    // Debug = 0
-    GenerateCoreDump = 1,           // Initiates core dump generation
-
-    ///////////////////////////////////////////////////////////////////////////
-    // EventPipe = 1024
-    StartEventPipeTracing = 1024,   // To file
-    StopEventPipeTracing,
-    CollectEventPipeTracing,        // To IPC
-
-    ///////////////////////////////////////////////////////////////////////////
-    // Profiler = 2048
-    AttachProfiler = 2048,
-};
-
-struct MessageHeader
-{
-    DiagnosticMessageType RequestType;
-    uint32_t Pid;
-};
+#include "diagnosticsprotocol.h"
 
 //! Defines an implementation of a IPC handler that dispatches messages to the runtime.
 class DiagnosticServer final
 {
 public:
-    //! Initialize the event pipe (Creates the EventPipe IPC server).
+    // Initialize the event pipe (Creates the EventPipe IPC server).
     static bool Initialize();
 
-    //! Shutdown the event pipe.
+    // Shutdown the event pipe.
     static bool Shutdown();
 
-    //! Diagnostics server thread.
+    // Diagnostics server thread.
     static DWORD WINAPI DiagnosticsServerThread(LPVOID lpThreadParameter);
 
+    // Pauses EEStartup after the Diagnostics Server has been started
+    // allowing a Diagnostics Monitor to attach perform tasks before
+    // Startup is completed
+    NOINLINE static void PauseForDiagnosticsMonitor();
+
+    // Sets CLREvent to resume startup in EEStartupHelper (see: ceemain.cpp after DiagnosticServer::Initialize() for pausing point)
+    // This is a no-op if not configured to pause or runtime has already resumed
+    static void ResumeRuntimeStartup();
+
 private:
     static Volatile<bool> s_shuttingDown;
+    static CLREventStatic *s_ResumeRuntimeStartupEvent;
+};
+
+class DiagnosticServerProtocolHelper
+{
+public:
+    // IPC event handlers.
+    static void HandleIpcMessage(DiagnosticsIpc::IpcMessage& message, IpcStream *pStream);
+    static void ResumeRuntimeStartup(DiagnosticsIpc::IpcMessage& message, IpcStream *pStream);
 };
 
 #endif // FEATURE_PERFTRACING

src/coreclr/src/vm/diagnosticsprotocol.h

@@ -75,10 +75,21 @@ namespace DiagnosticsIpc
         Server        = 0xFF,
     };
 
+    // Overlaps with DiagnosticServerResponseId
+    // DON'T create overlapping values
     enum class DiagnosticServerCommandId : uint8_t
     {
-        OK    = 0x00,
-        Error = 0xFF,
+        // 0x00 used in DiagnosticServerResponseId
+        ResumeRuntime = 0x01,
+        // 0xFF used DiagnosticServerResponseId
+    };
+
+    // Overlaps with DiagnosticServerCommandId
+    // DON'T create overlapping values
+    enum class DiagnosticServerResponseId : uint8_t
+    {
+        OK            = 0x00,
+        Error         = 0xFF,
     };
 
     struct MagicVersion
@@ -162,7 +173,7 @@ namespace DiagnosticsIpc
         { DotnetIpcMagic_V1 },
         (uint16_t)sizeof(IpcHeader),
         (uint8_t)DiagnosticServerCommandSet::Server,
-        (uint8_t)DiagnosticServerCommandId::OK,
+        (uint8_t)DiagnosticServerResponseId::OK,
         (uint16_t)0x0000
     };
 
@@ -171,7 +182,7 @@ namespace DiagnosticsIpc
         { DotnetIpcMagic_V1 },
         (uint16_t)sizeof(IpcHeader),
         (uint8_t)DiagnosticServerCommandSet::Server,
-        (uint8_t)DiagnosticServerCommandId::Error,
+        (uint8_t)DiagnosticServerResponseId::Error,
         (uint16_t)0x0000
     };
 

src/coreclr/src/vm/eventpipe.cpp

@@ -40,6 +40,8 @@ unsigned int * EventPipe::s_pProcGroupOffsets = nullptr;
 #endif
 Volatile<uint32_t> EventPipe::s_numberOfSessions(0);
 bool EventPipe::s_enableSampleProfilerAtStartup = false;
+CQuickArrayList<EventPipeSessionID> EventPipe::s_rgDeferredEventPipeSessionIds = CQuickArrayList<EventPipeSessionID>();
+bool EventPipe::s_CanStartThreads = false;
 
 // This function is auto-generated from /src/scripts/genEventPipe.py
 #ifdef TARGET_UNIX
@@ -109,6 +111,20 @@ void EventPipe::FinishInitialize()
 {
     STANDARD_VM_CONTRACT;
 
+    CrstHolder _crst(GetLock());
+
+    s_CanStartThreads = true;
+
+    while (s_rgDeferredEventPipeSessionIds.Size() > 0)
+    {
+        EventPipeSessionID id = s_rgDeferredEventPipeSessionIds.Pop();
+        if (IsSessionIdInCollection(id))
+        {
+            EventPipeSession *pSession = reinterpret_cast<EventPipeSession*>(id);
+            pSession->StartStreaming();
+        }
+    }
+
     if (s_enableSampleProfilerAtStartup)
     {
         SampleProfiler::Enable();
@@ -383,11 +399,15 @@ bool EventPipe::EnableInternal(
     // Enable tracing.
     s_config.Enable(*pSession, pEventPipeProviderCallbackDataQueue);
 
-    // Enable the sample profiler
-    if (enableSampleProfiler)
+    // Enable the sample profiler or defer until we can create threads
+    if (enableSampleProfiler && s_CanStartThreads)
     {
         SampleProfiler::Enable();
     }
+    else if (enableSampleProfiler && !s_CanStartThreads)
+    {
+        s_enableSampleProfilerAtStartup = true;
+    }
 
     return true;
 }
@@ -409,7 +429,15 @@ void EventPipe::StartStreaming(EventPipeSessionID id)
 
     EventPipeSession *const pSession = reinterpret_cast<EventPipeSession *>(id);
 
-    pSession->StartStreaming();
+    if (s_CanStartThreads)
+    {
+        pSession->StartStreaming();
+    }
+    else
+    {
+        s_rgDeferredEventPipeSessionIds.Push(id);
+    }
+
 }
 
 void EventPipe::Disable(EventPipeSessionID id)
@@ -422,7 +450,8 @@ void EventPipe::Disable(EventPipeSessionID id)
     }
     CONTRACTL_END;
 
-    SetupThread();
+    if (s_CanStartThreads)
+        SetupThread();
 
     if (id == 0)
         return;
@@ -492,7 +521,7 @@ void EventPipe::DisableInternal(EventPipeSessionID id, EventPipeProviderCallback
     pSession->Disable(); // WriteAllBuffersToFile, and remove providers.
 
     // Do rundown before fully stopping the session unless rundown wasn't requested
-    if (pSession->RundownRequested())
+    if (pSession->RundownRequested() && s_CanStartThreads)
     {
         pSession->EnableRundown(); // Set Rundown provider.
 

src/coreclr/src/vm/eventpipe.h

@@ -222,6 +222,10 @@ class EventPipe
     static Volatile<uint64_t> s_allowWrite;
     static EventPipeEventSource *s_pEventSource;
 
+    static bool s_CanStartThreads;
+
+    static CQuickArrayList<EventPipeSessionID> s_rgDeferredEventPipeSessionIds;
+
     //! Bitmask tracking EventPipe active sessions.
     // in all groups preceding it. For example if there are three groups with sizes:
     // 1, 7, 6 the table would be 0, 1, 8