ApplyStartupHook diagnostic IPC command (#3918)

Author: jander-msft

documentation/design-docs/ipc-protocol.md

@@ -380,6 +380,7 @@ enum class ProcessCommandId : uint8_t
     ResumeRuntime      = 0x01,
     ProcessEnvironment = 0x02,
     ProcessInfo2       = 0x04,
+    ApplyStartupHook   = 0x07
     // future
 }
 ```
@@ -845,6 +846,47 @@ struct Payload
 }
 ```
 
+### `ApplyStartupHook`
+
+Command Code: `0x0407`
+
+The `ApplyStartupHook` command is used to provide a path to a managed assembly with a [startup hook](https://github.com/dotnet/runtime/blob/main/docs/design/features/host-startup-hook.md) to the runtime. During diagnostic suspension, the startup hook path will be added list of hooks that the runtime will execute once it has been resumed.
+
+In the event of an [error](#Errors), the runtime will attempt to send an error message and subsequently close the connection.
+
+#### Inputs:
+
+Header: `{ Magic; Size; 0x0407; 0x0000 }`
+
+* `string startupHookPath`: The path to the managed assembly that contains the startup hook implementation.
+
+#### Returns (as an IPC Message Payload):
+
+Header: `{ Magic; size; 0xFF00; 0x0000; }`
+
+`ApplyStartupHook` returns:
+* `int32 hresult`: The result of adding the startup hook (`0` indicates success)
+
+##### Details:
+
+Input:
+```
+Payload
+{
+    string startupHookPath
+}
+```
+
+Returns:
+```c++
+struct Payload
+{
+    int32 hresult
+}
+```
+
+> Available since .NET 8.0
+
 ## Errors
 
 In the event an error occurs in the handling of an Ipc Message, the Diagnostic Server will attempt to send an Ipc Message encoding the error and subsequently close the connection.  The connection will be closed **regardless** of the success of sending the error message.  The Client is expected to be resilient in the event of a connection being abruptly closed.

src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/DiagnosticsClient.cs

@@ -298,6 +298,20 @@ internal async Task<Dictionary<string, string>> GetProcessEnvironmentAsync(Cance
             return await helper.ReadEnvironmentAsync(response.Continuation, token).ConfigureAwait(false);
         }
 
+        internal void ApplyStartupHook(string startupHookPath)
+        {
+            IpcMessage message = CreateApplyStartupHookMessage(startupHookPath);
+            IpcMessage response = IpcClient.SendMessage(_endpoint, message);
+            ValidateResponseMessage(response, nameof(ApplyStartupHook));
+        }
+
+        internal async Task ApplyStartupHookAsync(string startupHookPath, CancellationToken token)
+        {
+            IpcMessage message = CreateApplyStartupHookMessage(startupHookPath);
+            IpcMessage response = await IpcClient.SendMessageAsync(_endpoint, message, token).ConfigureAwait(false);
+            ValidateResponseMessage(response, nameof(ApplyStartupHookAsync));
+        }
+
         /// <summary>
         /// Get all the active processes that can be attached to.
         /// </summary>
@@ -576,6 +590,18 @@ private static IpcMessage CreateWriteDumpMessage(DumpCommandId command, DumpType
             return new IpcMessage(DiagnosticsServerCommandSet.Dump, (byte)command, payload);
         }
 
+        private static IpcMessage CreateApplyStartupHookMessage(string startupHookPath)
+        {
+            if (string.IsNullOrEmpty(startupHookPath))
+            {
+                throw new ArgumentException($"{nameof(startupHookPath)} required");
+            }
+
+            byte[] serializedConfiguration = SerializePayload(startupHookPath);
+
+            return new IpcMessage(DiagnosticsServerCommandSet.Process, (byte)ProcessCommandId.ApplyStartupHook, serializedConfiguration);
+        }
+
         private static ProcessInfo GetProcessInfoFromResponse(IpcResponse response, string operationName)
         {
             ValidateResponseMessage(response.Message, operationName);

src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsIpc/IpcCommands.cs

@@ -46,6 +46,7 @@ internal enum ProcessCommandId : byte
         ResumeRuntime = 0x01,
         GetProcessEnvironment = 0x02,
         SetEnvironmentVariable = 0x03,
-        GetProcessInfo2 = 0x04
+        GetProcessInfo2 = 0x04,
+        ApplyStartupHook = 0x07
     }
 }