Feature: mjpeg streaming for terminal control background

.github/actions/spell-check/dictionary/names.txt

@@ -4,6 +4,7 @@ Ballmer
 bhoj
 Bhojwani
 carlos
+Clauson
 dhowett
 Diviness
 dsafa

.github/actions/spell-check/whitelist/eacbc106c4603f6128d726ec0fdcb1cbfc6cab94.txt

@@ -0,0 +1,55 @@
+ABE
+bgra
+closesocket
+CSRC
+ctime
+deserialise
+deserialised
+DGRAM
+DHP
+dht
+DQT
+dri
+EEB
+EOI
+FDD
+ffmpeg
+hsample
+htonl
+htons
+huffman
+icase
+INADDR
+inband
+INET
+iphlpapi
+IPPROTO
+IPv
+IStorage
+jfif
+jpegtables
+libavcodec
+libavformat
+MAKEWORD
+mbz
+mjpeg
+posn
+premultiplied
+qtable
+recv
+recvfrom
+rtp
+rtpdec
+rtpsocket
+shutsdown
+sipsorcery
+sockaddr
+SOI
+stdint
+tcpip
+timeval
+usec
+vsample
+winsock
+wsa
+WSADATA

src/cascadia/TerminalControl/TermControl.cpp

@@ -11,9 +11,14 @@
 #include <WinUser.h>
 #include <LibraryResources.h>
 #include "..\..\types\inc\GlyphWidth.hpp"
+#include <regex>
+#include <string>
 
 #include "TermControl.g.cpp"
 #include "TermControlAutomationPeer.h"
+#include <winrt/Windows.Graphics.Imaging.h>
+#include <winrt/Windows.Storage.Pickers.h>
+#include <winrt/Windows.Storage.Streams.h>
 
 using namespace ::Microsoft::Console::Types;
 using namespace ::Microsoft::Terminal::Core;
@@ -369,7 +374,27 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 
         if (!_settings.BackgroundImage().empty())
         {
-            Windows::Foundation::Uri imageUri{ _settings.BackgroundImage() };
+            // Check if an MJPEG path has been requested.
+            std::string backgroundPath = to_string(_settings.BackgroundImage());
+            std::string regexInput = backgroundPath;
+            const std::regex pathRegex("(.*);mjpeg:(\\d+)$", std::regex_constants::icase);
+            std::smatch pathMatch;
+
+            if (std::regex_match(regexInput, pathMatch, pathRegex))
+            {
+                if (pathMatch.size() == 3)
+                {
+                    backgroundPath = pathMatch[1].str();
+                    int listenPort = std::stoi(pathMatch[2].str());
+
+                    _rtpSocket = std::make_unique<mjpeg::RtpSocket>(listenPort);
+                    auto fp = std::bind(&TermControl::_OnRtpFrameReady, this, std::placeholders::_1);
+                    _rtpSocket->SetFrameReadyCallback(fp);
+                }
+            }
+
+            // TODO: CHeck the MJPEG string checking doesn't lose Unicode chars.
+            Windows::Foundation::Uri imageUri{ to_hstring(backgroundPath) };
 
             // Check if the image brush is already pointing to the image
             // in the modified settings; if it isn't (or isn't there),
@@ -393,6 +418,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
             BackgroundImage().Opacity(_settings.BackgroundImageOpacity());
             BackgroundImage().HorizontalAlignment(_settings.BackgroundImageHorizontalAlignment());
             BackgroundImage().VerticalAlignment(_settings.BackgroundImageVerticalAlignment());
+
+            if (_rtpSocket != nullptr)
+            {
+                _rtpSocket->Start();
+            }
         }
         else
         {
@@ -1528,6 +1558,42 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         }
     }
 
+    // Method Description:
+    // Callback function for MJPEG receiver when a full JPEG frame is ready.
+    void TermControl::_OnRtpFrameReady(std::vector<uint8_t>& jpeg)
+    {
+        _SetBackgroundImage(jpeg);
+    }
+
+    // Method Description:
+    // Sets the terminal control background to the JPEG image contained in the supplied
+    // vector.
+    winrt::fire_and_forget TermControl::_SetBackgroundImage(std::vector<uint8_t> jpeg)
+    {
+        co_await winrt::resume_foreground(Dispatcher());
+
+        Windows::Storage::Streams::InMemoryRandomAccessStream jpegStream;
+        auto writer = Windows::Storage::Streams::DataWriter(jpegStream.GetOutputStreamAt(0));
+        writer.WriteBytes(jpeg);
+        co_await writer.StoreAsync();
+
+        auto decoder = co_await Windows::Graphics::Imaging::BitmapDecoder::CreateAsync(jpegStream);
+
+        auto bmp = co_await decoder.GetSoftwareBitmapAsync();
+
+        if (bmp.BitmapPixelFormat() != Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8 ||
+            bmp.BitmapAlphaMode() == Windows::Graphics::Imaging::BitmapAlphaMode::Straight)
+        {
+            bmp = Windows::Graphics::Imaging::SoftwareBitmap::Convert(bmp,
+                                                                      Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8,
+                                                                      Windows::Graphics::Imaging::BitmapAlphaMode::Premultiplied);
+        }
+
+        Windows::UI::Xaml::Media::Imaging::SoftwareBitmapSource source;
+        co_await source.SetBitmapAsync(bmp);
+        BackgroundImage().Source(source);
+    }
+
     // Method Description:
     // - Event handler for the GotFocus event. This is used to...
     //   - enable accessibility notifications for this TermControl
@@ -2135,6 +2201,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
             TSFInputControl().Close(); // Disconnect the TSF input control so it doesn't receive EditContext events.
             _autoScrollTimer.Stop();
 
+            if (_rtpSocket != nullptr)
+            {
+                _rtpSocket->Close();
+            }
+
             // GH#1996 - Close the connection asynchronously on a background
             // thread.
             // Since TermControl::Close is only ever triggered by the UI, we

src/cascadia/TerminalControl/TermControl.h

@@ -15,6 +15,7 @@
 #include "../buffer/out/search.h"
 #include "cppwinrt_utils.h"
 #include "SearchBoxControl.h"
+#include "rtpsocket.h"
 
 namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 {
@@ -242,6 +243,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         // Unbounded main dispatcher use leads to massive memory leaks and intense slowdowns
         // on the UI thread.
         std::atomic<bool> _coroutineDispatchStateUpdateInProgress{ false };
+
+        // Socket and callbacks used for MJPEG background on terminal control.
+        std::unique_ptr<mjpeg::RtpSocket> _rtpSocket{ nullptr };
+        void _OnRtpFrameReady(std::vector<uint8_t>& jpeg);
+        winrt::fire_and_forget _SetBackgroundImage(std::vector<uint8_t> jpeg);
     };
 }
 

src/cascadia/TerminalControl/TerminalControl.vcxproj

@@ -19,7 +19,6 @@
          projects compile properly when they depend on this "Microsoft.winmd."
     -->
     <CppWinRTNamespaceMergeDepth>3</CppWinRTNamespaceMergeDepth>
-
     <!--
       DON'T REDIRECT OUR OUTPUT.
       Setting this will tell cppwinrt.build.post.props to copy our output from
@@ -28,12 +27,12 @@
     <NoOutputRedirection>true</NoOutputRedirection>
     <XamlComponentResourceLocation>nested</XamlComponentResourceLocation>
   </PropertyGroup>
-
   <Import Project="..\..\..\common.openconsole.props" Condition="'$(OpenConsoleDir)'==''" />
   <Import Project="$(OpenConsoleDir)src\cppwinrt.build.pre.props" />
-
   <ItemGroup>
+    <ClInclude Include="mjpeg.h" />
     <ClInclude Include="pch.h" />
+    <ClInclude Include="rtpsocket.h" />
     <ClInclude Include="SearchBoxControl.h">
       <DependentUpon>SearchBoxControl.xaml</DependentUpon>
     </ClInclude>
@@ -53,6 +52,7 @@
       <PrecompiledHeader>Create</PrecompiledHeader>
     </ClCompile>
     <ClCompile Include="init.cpp" />
+    <ClCompile Include="rtpsocket.cpp" />
     <ClCompile Include="SearchBoxControl.cpp">
       <DependentUpon>SearchBoxControl.xaml</DependentUpon>
     </ClCompile>
@@ -122,9 +122,7 @@
       <SubType>Designer</SubType>
     </Page>
   </ItemGroup>
-
   <Import Project="$(OpenConsoleDir)src\cppwinrt.build.post.props" />
-
   <ItemDefinitionGroup>
     <Link>
       <AdditionalDependencies>dwrite.lib;dxgi.lib;d2d1.lib;d3d11.lib;shcore.lib;winmm.lib;pathcch.lib;propsys.lib;uiautomationcore.lib;Shlwapi.lib;ntdll.lib;user32.lib;kernel32.lib;%(AdditionalDependencies)</AdditionalDependencies>
@@ -133,4 +131,4 @@
       <AdditionalIncludeDirectories>$(OpenConsoleDir)src\cascadia\inc;$(OpenConsoleDir)src\types\inc;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
   </ItemDefinitionGroup>
-</Project>
+</Project>

src/cascadia/TerminalControl/TerminalControl.vcxproj.filters

@@ -15,31 +15,33 @@
     <ClCompile Include="$(GeneratedFilesDir)module.g.cpp" />
     <ClCompile Include="TermControlAutomationPeer.cpp" />
     <ClCompile Include="XamlUiaTextRange.cpp" />
-    <ClCompile Include="TermControlUiaProvider.cpp" />
-    <ClCompile Include="UiaTextRange.cpp" />
     <ClCompile Include="SearchBoxControl.cpp" />
     <ClCompile Include="init.cpp" />
+    <ClCompile Include="rtpsocket.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="pch.h" />
     <ClInclude Include="TermControl.h" />
     <ClInclude Include="TermControlAutomationPeer.h" />
     <ClInclude Include="XamlUiaTextRange.h" />
-    <ClInclude Include="TermControlUiaProvider.hpp" />
-    <ClInclude Include="UiaTextRange.hpp" />
+    <ClInclude Include="rtpsocket.h" />
+    <ClInclude Include="mjpeg.h" />
   </ItemGroup>
   <ItemGroup>
     <Midl Include="TermControl.idl" />
     <Midl Include="TermControlAutomationPeer.idl" />
     <Midl Include="SearchBoxControl.idl" />
     <Midl Include="TSFInputControl.idl" />
+    <Midl Include="IMouseWheelListener.idl" />
   </ItemGroup>
   <ItemGroup>
     <None Include="TerminalControl.def" />
     <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
     <Page Include="SearchBoxControl.xaml" />
+    <Page Include="TermControl.xaml" />
+    <Page Include="TSFInputControl.xaml" />
   </ItemGroup>
   <ItemGroup>
     <Natvis Include="$(SolutionDir)tools\ConsoleTypes.natvis" />
@@ -49,4 +51,4 @@
       <Filter>Resources</Filter>
     </PRIResource>
   </ItemGroup>
-</Project>
+</Project>