Reintroduce a color compatibility hack, but only for PowerShells

src/buffer/out/TextAttribute.cpp

@@ -20,6 +20,47 @@ void TextAttribute::SetLegacyDefaultAttributes(const WORD defaultAttributes) noe
     s_legacyDefaultBackground = (defaultAttributes & BG_ATTRS) >> 4;
 }
 
+// Routine Description:
+// Pursuant to GH#6807
+// This routine replaces VT colors from the 16-color set with the "default"
+// flag. It is intended to be used as part of the "VT Quirk" in
+// WriteConsole[AW].
+//
+// There is going to be a very long tail of applications that will
+// explicitly request VT SGR 40/37 when what they really want is to
+// SetConsoleTextAttribute() with a black background/white foreground.
+// Instead of making those applications look bad (and therefore making us
+// look bad, because we're releasing this as an update to something that
+// "looks good" already), we're introducing this compatibility hack. Before
+// the color reckoning in GH#6698 + GH#6506, *every* color was subject to
+// being spontaneously and erroneously turned into the default color. Now,
+// only the 16-color palette value that matches the active console
+// background color will be destroyed when the quirk is enabled.
+//
+// This is not intended to be a long-term solution. This comment will be
+// discovered in forty years(*) time and people will laugh at our hubris.
+//
+// *it doesn't matter when you're reading this, it will always be 40 years
+// from now.
+TextAttribute TextAttribute::StripErroneousVT16VersionsOfLegacyDefaults(const TextAttribute& attribute) noexcept
+{
+    const auto fg{ attribute.GetForeground() };
+    const auto bg{ attribute.GetBackground() };
+    auto copy{ attribute };
+    if (fg.IsIndex16() &&
+        attribute.IsBold() == WI_IsFlagSet(s_legacyDefaultForeground, FOREGROUND_INTENSITY) &&
+        fg.GetIndex() == (s_legacyDefaultForeground & ~FOREGROUND_INTENSITY))
+    {
+        // We don't want to turn 1;37m into 39m (or even 1;39m), as this was meant to mimic a legacy color.
+        copy.SetDefaultForeground();
+    }
+    if (bg.IsIndex16() && bg.GetIndex() == s_legacyDefaultBackground)
+    {
+        copy.SetDefaultBackground();
+    }
+    return copy;
+}
+
 // Routine Description:
 // - Returns a WORD with legacy-style attributes for this textattribute.
 // Parameters:

src/buffer/out/TextAttribute.hpp

@@ -60,6 +60,7 @@ class TextAttribute final
     }
 
     static void SetLegacyDefaultAttributes(const WORD defaultAttributes) noexcept;
+    static TextAttribute StripErroneousVT16VersionsOfLegacyDefaults(const TextAttribute& attribute) noexcept;
     WORD GetLegacyAttributes() const noexcept;
 
     COLORREF CalculateRgbForeground(std::basic_string_view<COLORREF> colorTable,

src/host/ApiRoutines.h

@@ -98,11 +98,13 @@ class ApiRoutines : public IApiRoutines
     [[nodiscard]] HRESULT WriteConsoleAImpl(IConsoleOutputObject& context,
                                             const std::string_view buffer,
                                             size_t& read,
+                                            bool requiresVtQuirk,
                                             std::unique_ptr<IWaitRoutine>& waiter) noexcept override;
 
     [[nodiscard]] HRESULT WriteConsoleWImpl(IConsoleOutputObject& context,
                                             const std::wstring_view buffer,
                                             size_t& read,
+                                            bool requiresVtQuirk,
                                             std::unique_ptr<IWaitRoutine>& waiter) noexcept override;
 
 #pragma region ThreadCreationInfo

src/host/_stream.cpp

@@ -971,6 +971,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
 [[nodiscard]] NTSTATUS DoWriteConsole(_In_reads_bytes_(*pcbBuffer) PWCHAR pwchBuffer,
                                       _Inout_ size_t* const pcbBuffer,
                                       SCREEN_INFORMATION& screenInfo,
+                                      bool requiresVtQuirk,
                                       std::unique_ptr<WriteData>& waiter)
 {
     const CONSOLE_INFORMATION& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
@@ -981,7 +982,8 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
             waiter = std::make_unique<WriteData>(screenInfo,
                                                  pwchBuffer,
                                                  *pcbBuffer,
-                                                 gci.OutputCP);
+                                                 gci.OutputCP,
+                                                 requiresVtQuirk);
         }
         catch (...)
         {
@@ -991,6 +993,19 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
         return CONSOLE_STATUS_WAIT;
     }
 
+    auto restoreVtQuirk{
+        wil::scope_exit([&]() { screenInfo.ResetIgnoreLegacyEquivalentVTAttributes(); })
+    };
+
+    if (requiresVtQuirk)
+    {
+        screenInfo.SetIgnoreLegacyEquivalentVTAttributes();
+    }
+    else
+    {
+        restoreVtQuirk.release();
+    }
+
     const auto& textBuffer = screenInfo.GetTextBuffer();
     return WriteChars(screenInfo,
                       pwchBuffer,
@@ -1020,6 +1035,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
 [[nodiscard]] HRESULT WriteConsoleWImplHelper(IConsoleOutputObject& context,
                                               const std::wstring_view buffer,
                                               size_t& read,
+                                              bool requiresVtQuirk,
                                               std::unique_ptr<WriteData>& waiter) noexcept
 {
     try
@@ -1032,7 +1048,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
         size_t cbTextBufferLength;
         RETURN_IF_FAILED(SizeTMult(buffer.size(), sizeof(wchar_t), &cbTextBufferLength));
 
-        NTSTATUS Status = DoWriteConsole(const_cast<wchar_t*>(buffer.data()), &cbTextBufferLength, context, waiter);
+        NTSTATUS Status = DoWriteConsole(const_cast<wchar_t*>(buffer.data()), &cbTextBufferLength, context, requiresVtQuirk, waiter);
 
         // Convert back from bytes to characters for the resulting string length written.
         read = cbTextBufferLength / sizeof(wchar_t);
@@ -1065,6 +1081,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
 [[nodiscard]] HRESULT ApiRoutines::WriteConsoleAImpl(IConsoleOutputObject& context,
                                                      const std::string_view buffer,
                                                      size_t& read,
+                                                     bool requiresVtQuirk,
                                                      std::unique_ptr<IWaitRoutine>& waiter) noexcept
 {
     try
@@ -1176,7 +1193,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
 
         // Make the W version of the call
         size_t wcBufferWritten{};
-        const auto hr{ WriteConsoleWImplHelper(screenInfo, wstr, wcBufferWritten, writeDataWaiter) };
+        const auto hr{ WriteConsoleWImplHelper(screenInfo, wstr, wcBufferWritten, requiresVtQuirk, writeDataWaiter) };
 
         // If there is no waiter, process the byte count now.
         if (nullptr == writeDataWaiter.get())
@@ -1254,6 +1271,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
 [[nodiscard]] HRESULT ApiRoutines::WriteConsoleWImpl(IConsoleOutputObject& context,
                                                      const std::wstring_view buffer,
                                                      size_t& read,
+                                                     bool requiresVtQuirk,
                                                      std::unique_ptr<IWaitRoutine>& waiter) noexcept
 {
     try
@@ -1262,7 +1280,7 @@ constexpr unsigned int LOCAL_BUFFER_SIZE = 100;
         auto unlock = wil::scope_exit([&] { UnlockConsole(); });
 
         std::unique_ptr<WriteData> writeDataWaiter;
-        RETURN_IF_FAILED(WriteConsoleWImplHelper(context.GetActiveBuffer(), buffer, read, writeDataWaiter));
+        RETURN_IF_FAILED(WriteConsoleWImplHelper(context.GetActiveBuffer(), buffer, read, requiresVtQuirk, writeDataWaiter));
 
         // Transfer specific waiter pointer into the generic interface wrapper.
         waiter.reset(writeDataWaiter.release());

src/host/_stream.h

@@ -93,4 +93,5 @@ Return Value:
 [[nodiscard]] NTSTATUS DoWriteConsole(_In_reads_bytes_(*pcbBuffer) PWCHAR pwchBuffer,
                                       _Inout_ size_t* const pcbBuffer,
                                       SCREEN_INFORMATION& screenInfo,
+                                      bool requiresVtQuirk,
                                       std::unique_ptr<WriteData>& waiter);

src/host/screenInfo.cpp

@@ -58,7 +58,8 @@ SCREEN_INFORMATION::SCREEN_INFORMATION(
     _virtualBottom{ 0 },
     _renderTarget{ *this },
     _currentFont{ fontInfo },
-    _desiredFont{ fontInfo }
+    _desiredFont{ fontInfo },
+    _ignoreLegacyEquivalentVTAttributes{ false }
 {
     // Check if VT mode is enabled. Note that this can be true w/o calling
     // SetConsoleMode, if VirtualTerminalLevel is set to !=0 in the registry.
@@ -2029,6 +2030,13 @@ TextAttribute SCREEN_INFORMATION::GetPopupAttributes() const
 // <none>
 void SCREEN_INFORMATION::SetAttributes(const TextAttribute& attributes)
 {
+    if (_ignoreLegacyEquivalentVTAttributes)
+    {
+        // See the comment on StripErroneousVT16VersionsOfLegacyDefaults for more info.
+        _textBuffer->SetCurrentAttributes(TextAttribute::StripErroneousVT16VersionsOfLegacyDefaults(attributes));
+        return;
+    }
+
     _textBuffer->SetCurrentAttributes(attributes);
 
     // If we're an alt buffer, DON'T propagate this setting up to the main buffer.
@@ -2675,3 +2683,17 @@ Viewport SCREEN_INFORMATION::GetScrollingRegion() const noexcept
                                                   marginsSet ? marginRect.Bottom : buffer.BottomInclusive() });
     return margin;
 }
+
+// Routine Description:
+// - Engages the legacy VT handling quirk; see TextAttribute::StripErroneousVT16VersionsOfLegacyDefaults
+void SCREEN_INFORMATION::SetIgnoreLegacyEquivalentVTAttributes() noexcept
+{
+    _ignoreLegacyEquivalentVTAttributes = true;
+}
+
+// Routine Description:
+// - Disengages the legacy VT handling quirk; see TextAttribute::StripErroneousVT16VersionsOfLegacyDefaults
+void SCREEN_INFORMATION::ResetIgnoreLegacyEquivalentVTAttributes() noexcept
+{
+    _ignoreLegacyEquivalentVTAttributes = false;
+}

src/host/screenInfo.hpp

@@ -238,6 +238,9 @@ class SCREEN_INFORMATION : public ConsoleObjectHeader, public Microsoft::Console
 
     void InitializeCursorRowAttributes();
 
+    void SetIgnoreLegacyEquivalentVTAttributes() noexcept;
+    void ResetIgnoreLegacyEquivalentVTAttributes() noexcept;
+
 private:
     SCREEN_INFORMATION(_In_ Microsoft::Console::Interactivity::IWindowMetrics* pMetrics,
                        _In_ Microsoft::Console::Interactivity::IAccessibilityNotifier* pNotifier,
@@ -300,6 +303,8 @@ class SCREEN_INFORMATION : public ConsoleObjectHeader, public Microsoft::Console
 
     ScreenBufferRenderTarget _renderTarget;
 
+    bool _ignoreLegacyEquivalentVTAttributes;
+
 #ifdef UNIT_TESTING
     friend class TextBufferIteratorTests;
     friend class ScreenBufferTests;

src/host/ut_host/ApiRoutinesTests.cpp

@@ -386,7 +386,7 @@ class ApiRoutinesTests
             const size_t cchWriteLength = std::min(cchIncrement, cchTestText - i);
 
             // Run the test method
-            const HRESULT hr = _pApiRoutines->WriteConsoleAImpl(si, { pszTestText + i, cchWriteLength }, cchRead, waiter);
+            const HRESULT hr = _pApiRoutines->WriteConsoleAImpl(si, { pszTestText + i, cchWriteLength }, cchRead, false, waiter);
 
             VERIFY_ARE_EQUAL(S_OK, hr, L"Successful result code from writing.");
             if (!fInduceWait)
@@ -442,7 +442,7 @@ class ApiRoutinesTests
 
         size_t cchRead = 0;
         std::unique_ptr<IWaitRoutine> waiter;
-        const HRESULT hr = _pApiRoutines->WriteConsoleWImpl(si, testText, cchRead, waiter);
+        const HRESULT hr = _pApiRoutines->WriteConsoleWImpl(si, testText, cchRead, false, waiter);
 
         VERIFY_ARE_EQUAL(S_OK, hr, L"Successful result code from writing.");
         if (!fInduceWait)