Merged PR 3528883: Force the use of v2 (non-legacy) conhost when in C…

…onPTY mode (#1935)

Force the use of v2 (non-legacy) conhost when in ConPTY mode (#1935)

Fixes #1838.

Related work items: #22981851

src/host/exemain.cpp

@@ -16,7 +16,7 @@ TRACELOGGING_DEFINE_PROVIDER(
     (0x770aa552, 0x671a, 0x5e97, 0x57, 0x9b, 0x15, 0x17, 0x09, 0xec, 0x0d, 0xbd),
     TraceLoggingOptionMicrosoftTelemetry());
 
-static bool ShouldUseConhostV2()
+static bool ConhostV2ForcedInRegistry()
 {
     // If the registry value doesn't exist, or exists and is non-zero, we should default to using the v2 console.
     // Otherwise, in the case of an explicit value of 0, we should use the legacy console.
@@ -83,9 +83,21 @@ static bool ShouldUseConhostV2()
     }
 }
 
-static bool ShouldUseLegacyConhost(const bool fForceV1)
+static bool ShouldUseLegacyConhost(const ConsoleArguments& args)
 {
-    return fForceV1 || !ShouldUseConhostV2();
+    if (args.InConptyMode())
+    {
+        return false;
+    }
+
+    if (args.GetForceV1())
+    {
+        return true;
+    }
+
+    // Per the documentation in ConhostV2ForcedInRegistry, it checks the value
+    // of HKCU\Console:ForceV2. If it's *not found* or nonzero, "v2" is forced.
+    return !ConhostV2ForcedInRegistry();
 }
 
 [[nodiscard]] static HRESULT ActivateLegacyConhost(const HANDLE handle)
@@ -163,7 +175,7 @@ int CALLBACK wWinMain(
     HRESULT hr = args.ParseCommandline();
     if (SUCCEEDED(hr))
     {
-        if (ShouldUseLegacyConhost(args.GetForceV1()))
+        if (ShouldUseLegacyConhost(args))
         {
             if (args.ShouldCreateServerHandle())
             {