Enabling native code debugging of test host (#2190)

* Enabling native code debugging of test host * Suggested changes addressed
microsoft · Sep 25, 2019 · 462d367 · 462d367
1 parent 8e6f70f
commit 462d367
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 15 deletions.
diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs
@@ -147,7 +147,11 @@ public virtual bool SetupChannel(IEnumerable<string> sources, string runSettings
 
                 // Warn the user that execution will wait for debugger attach.
                 var hostDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_DEBUG");
-                if (!string.IsNullOrEmpty(hostDebugEnabled) && hostDebugEnabled.Equals("1", StringComparison.Ordinal))
+                var nativeHostDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_NATIVE_DEBUG");
+
+                if (!string.IsNullOrEmpty(hostDebugEnabled) && hostDebugEnabled.Equals("1", StringComparison.Ordinal) ||
+                    new PlatformEnvironment().OperatingSystem.Equals(PlatformOperatingSystem.Windows) && 
+                    !string.IsNullOrEmpty(nativeHostDebugEnabled) && nativeHostDebugEnabled.Equals("1", StringComparison.Ordinal))
                 {
                     ConsoleOutput.Instance.WriteLine(CrossPlatEngineResources.HostDebuggerWarning, OutputLevel.Warning);
                     ConsoleOutput.Instance.WriteLine(

diff --git a/src/testhost.x86/Program.cs b/src/testhost.x86/Program.cs
@@ -7,6 +7,7 @@ namespace Microsoft.VisualStudio.TestPlatform.TestHost
     using System.Collections.Generic;
     using System.Diagnostics;
     using System.Globalization;
+    using System.Runtime.InteropServices;
     using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers;
     using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
@@ -83,25 +84,34 @@ private static IEngineInvoker GetEngineInvoker(IDictionary<string, string> argsD
         }
 
         private static void WaitForDebuggerIfEnabled()
-        {
-            IProcessHelper processHelper = new ProcessHelper();
-            var debugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_DEBUG");
-            if (!string.IsNullOrEmpty(debugEnabled) && debugEnabled.Equals("1", StringComparison.Ordinal))
-            {
-                ConsoleOutput.Instance.WriteLine("Waiting for debugger attach...", OutputLevel.Information);
-
-                var currentProcessId = processHelper.GetCurrentProcessId();
-                var currentProcessName = processHelper.GetProcessName(currentProcessId);
-                ConsoleOutput.Instance.WriteLine(
-                    string.Format("Process Id: {0}, Name: {1}", currentProcessId, currentProcessName),
-                    OutputLevel.Information);
+        {   
+            // Check if native debugging is enabled and OS is windows.
+            var nativeDebugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_NATIVE_DEBUG");
 
-                while (!Debugger.IsAttached)
+            if (!string.IsNullOrEmpty(nativeDebugEnabled) && nativeDebugEnabled.Equals("1", StringComparison.Ordinal)
+                && new PlatformEnvironment().OperatingSystem.Equals(PlatformOperatingSystem.Windows))
+            {
+                while (!IsDebuggerPresent())
                 {
                     System.Threading.Tasks.Task.Delay(1000).Wait();
                 }
 
-                Debugger.Break();
+                DebugBreak();
+            }
+            // else check for host debugging enabled
+            else
+            {
+                var debugEnabled = Environment.GetEnvironmentVariable("VSTEST_HOST_DEBUG");
+
+                if (!string.IsNullOrEmpty(debugEnabled) && debugEnabled.Equals("1", StringComparison.Ordinal))
+                {
+                    while (!Debugger.IsAttached)
+                    {
+                        System.Threading.Tasks.Task.Delay(1000).Wait();
+                    }
+
+                    Debugger.Break();
+                }
             }
         }
 
@@ -120,5 +130,13 @@ private static void SetCultureSpecifiedByUser()
                 }
             }
         }
+
+        // Native APIs for enabling native debugging.
+        [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        internal static extern bool IsDebuggerPresent();
+
+        [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)]
+        internal static extern void DebugBreak();
     }
 }