Merge pull request #63 from jdibenes/pv_mrc

Support for Mixed Reality Capture

README.md

@@ -31,6 +31,7 @@ HoloLens 2 server software and Python client library for streaming sensor data v
 
 - Download calibration data (e.g., camera intrinsics, extrinsics, undistort maps) for the Front Camera and Research Mode sensors (except RM IMU Magnetometer).
 - Optional per-frame pose for the Front Camera and Research Mode sensors.
+- Support for Mixed Reality Capture (Holograms in Front Camera video).
 - Client can configure the bitrate and properties of the [H264](https://learn.microsoft.com/en-us/windows/win32/medfound/h-264-video-encoder), [HEVC](https://learn.microsoft.com/en-us/windows/win32/medfound/h-265---hevc-video-encoder), and [AAC](https://learn.microsoft.com/en-us/windows/win32/medfound/aac-encoder) encoded streams.
 - Client can configure the resolution and framerate of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/etc/pv_configurations.txt) for a list of supported configurations.
 - Client can configure the focus, white balance, and exposure of the Front Camera. See [here](https://github.com/jdibenes/hl2ss/blob/main/viewer/client_ipc_rc.py).

hl2ss/hl2ss/Package.appxmanifest

@@ -9,7 +9,7 @@
   <Identity
     Name="eaaf3af3-1402-4e5b-b6a1-5d0fbb7c1ba8"
     Publisher="CN=jcds"
-    Version="1.0.23.0" />
+    Version="1.0.24.0" />
   <mp:PhoneIdentity PhoneProductId="eaaf3af3-1402-4e5b-b6a1-5d0fbb7c1ba8" PhonePublisherId="00000000-0000-0000-0000-000000000000"/>
   <Properties>
     <DisplayName>hl2ss</DisplayName>

hl2ss/hl2ss/custom_video_effect.cpp

@@ -0,0 +1,30 @@
+
+#include "custom_video_effect.h"
+
+using namespace winrt::Windows::Foundation::Collections;
+using namespace winrt::Windows::Media::Effects;
+
+MRCVideoEffect::MRCVideoEffect(MRCOptions const& options)
+{
+    m_propertySet.Insert(L"StreamType", winrt::box_value<uint32_t>(1));
+    m_propertySet.Insert(L"HologramCompositionEnabled", winrt::box_value<bool>(options.hologram_composition));
+    m_propertySet.Insert(L"RecordingIndicatorEnabled", winrt::box_value<bool>(options.recording_indicator));
+    m_propertySet.Insert(L"VideoStabilizationEnabled", winrt::box_value<bool>(options.video_stabilization));
+    m_propertySet.Insert(L"VideoStabilizationBufferLength", winrt::box_value<uint32_t>(options.video_stabilization_length));
+    m_propertySet.Insert(L"GlobalOpacityCoefficient", winrt::box_value<float>(options.global_opacity));
+    m_propertySet.Insert(L"BlankOnProtectedContent", winrt::box_value<bool>(options.blank_protected));
+    m_propertySet.Insert(L"ShowHiddenMesh", winrt::box_value<bool>(options.show_mesh));
+    m_propertySet.Insert(L"OutputSize", winrt::box_value(winrt::Windows::Foundation::Size(options.output_width, options.output_height)));
+    m_propertySet.Insert(L"OutputSubtype", winrt::box_value(winrt::hstring(L"Nv12")));
+    m_propertySet.Insert(L"PreferredHologramPerspective", winrt::box_value<uint32_t>(options.hologram_perspective));
+}
+
+IPropertySet MRCVideoEffect::Properties()
+{
+    return m_propertySet;
+}
+
+winrt::hstring MRCVideoEffect::ActivatableClassId()
+{
+    return L"Windows.Media.MixedRealityCapture.MixedRealityCaptureVideoEffect";
+}

hl2ss/hl2ss/custom_video_effect.h

@@ -0,0 +1,32 @@
+
+#pragma once
+
+#include <winrt/Windows.Foundation.h>
+#include <winrt/Windows.Foundation.Collections.h>
+#include <winrt/Windows.Media.Effects.h>
+
+struct MRCOptions
+{
+    bool enable;
+    bool hologram_composition;
+    bool recording_indicator;
+    bool video_stabilization;
+    bool blank_protected;
+    bool show_mesh;
+    uint8_t _reserved[2];
+    float global_opacity;
+    float output_width;
+    float output_height;
+    uint32_t video_stabilization_length;
+    uint32_t hologram_perspective;
+};
+
+struct MRCVideoEffect : winrt::implements<MRCVideoEffect, winrt::Windows::Media::Effects::IVideoEffectDefinition>
+{
+    winrt::Windows::Foundation::Collections::PropertySet m_propertySet;
+
+    MRCVideoEffect(MRCOptions const& options);
+
+    winrt::hstring ActivatableClassId();
+    winrt::Windows::Foundation::Collections::IPropertySet Properties();
+};

hl2ss/hl2ss/hl2ss.vcxproj

@@ -190,6 +190,7 @@
     <ClInclude Include="custom_sink_writers.h" />
     <ClInclude Include="custom_stream_sink.h" />
     <ClInclude Include="custom_hook_callback.h" />
+    <ClInclude Include="custom_video_effect.h" />
     <ClInclude Include="display7s.h" />
     <ClInclude Include="extended_eye_tracking.h" />
     <ClInclude Include="holographic_space.h" />
@@ -264,6 +265,7 @@
     <ClCompile Include="custom_sink_writers.cpp" />
     <ClCompile Include="custom_stream_sink.cpp" />
     <ClCompile Include="custom_hook_callback.cpp" />
+    <ClCompile Include="custom_video_effect.cpp" />
     <ClCompile Include="display7s.cpp" />
     <ClCompile Include="extended_eye_tracking.cpp" />
     <ClCompile Include="holographic_space.cpp" />

hl2ss/hl2ss/hl2ss.vcxproj.filters

@@ -217,6 +217,9 @@
     <ClCompile Include="..\..\3rdparty\Zdepth\zstd\src\zstdmt_compress.c">
       <Filter>Libraries\Zdepth\zstd</Filter>
     </ClCompile>
+    <ClCompile Include="custom_video_effect.cpp">
+      <Filter>Source Files\sink</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Image Include="Assets\Wide310x150Logo.scale-200.png">
@@ -561,6 +564,9 @@
     <ClInclude Include="..\..\3rdparty\Zdepth\zstd\include\zstd.h">
       <Filter>Libraries\Zdepth\zstd</Filter>
     </ClInclude>
+    <ClInclude Include="custom_video_effect.h">
+      <Filter>Header Files\sink</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <AppxManifest Include="Package.appxmanifest">

hl2ss/hl2ss/ipc_sc.cpp

@@ -2,6 +2,7 @@
 #include "server.h"
 #include "custom_media_types.h"
 #include "custom_sink_writers.h"
+#include "custom_video_effect.h"
 #include "ipc_sc.h"
 
 #include <winrt/Windows.Graphics.Imaging.h>
@@ -168,3 +169,34 @@ bool ReceiveZABFormat_Profile(SOCKET clientsocket, ZABFormat& format)
 
 	return true;
 }
+
+// OK
+bool ReceiveMRCOptions(SOCKET clientsocket, MRCOptions& options)
+{
+	bool ok;
+
+	ok = recv_u8(clientsocket, *(uint8_t*)&options.enable);
+	if (!ok) { return false; }
+	ok = recv_u8(clientsocket, *(uint8_t*)&options.hologram_composition);
+	if (!ok) { return false; }
+	ok = recv_u8(clientsocket, *(uint8_t*)&options.recording_indicator);
+	if (!ok) { return false; }
+	ok = recv_u8(clientsocket, *(uint8_t*)&options.video_stabilization);
+	if (!ok) { return false; }
+	ok = recv_u8(clientsocket, *(uint8_t*)&options.blank_protected);
+	if (!ok) { return false; }
+	ok = recv_u8(clientsocket, *(uint8_t*)&options.show_mesh);
+	if (!ok) { return false; }
+	ok = recv_u32(clientsocket, *(uint32_t*)&options.global_opacity);
+	if (!ok) { return false; }
+	ok = recv_u32(clientsocket, *(uint32_t*)&options.output_width);
+	if (!ok) { return false; }
+	ok = recv_u32(clientsocket, *(uint32_t*)&options.output_height);
+	if (!ok) { return false; }
+	ok = recv_u32(clientsocket, options.video_stabilization_length);
+	if (!ok) { return false; }
+	ok = recv_u32(clientsocket, options.hologram_perspective);
+	if (!ok) { return false; }
+
+	return true;
+}

hl2ss/hl2ss/ipc_sc.h

@@ -4,6 +4,7 @@
 #include "server.h"
 #include "custom_media_types.h"
 #include "custom_sink_writers.h"
+#include "custom_video_effect.h"
 
 #include <winrt/Windows.Graphics.Imaging.h>
 
@@ -27,3 +28,4 @@ bool ReceiveH26xFormat_Profile(SOCKET clientsocket, H26xFormat& format);
 bool ReceiveH26xEncoder_Options(SOCKET clientsocket, std::vector<uint64_t>& options);
 bool ReceiveZABFormat_PNGFilter(SOCKET clientsocket, ZABFormat& format);
 bool ReceiveZABFormat_Profile(SOCKET clientsocket, ZABFormat& format);
+bool ReceiveMRCOptions(SOCKET clientsocket, MRCOptions& options);

hl2ss/hl2ss/personal_video.cpp

@@ -1,5 +1,6 @@
 
 #include "lock.h"
+#include "custom_video_effect.h"
 
 #include <winrt/Windows.Foundation.Collections.h>
 #include <winrt/Windows.Media.MediaProperties.h>
@@ -93,7 +94,7 @@ void PersonalVideo_Cleanup()
 }
 
 // OK
-void PersonalVideo_Open()
+void PersonalVideo_Open(MRCOptions const& options)
 {
     uint32_t const width     = 1920;
     uint32_t const height    = 1080;
@@ -121,6 +122,8 @@ void PersonalVideo_Open()
 
     g_mediaCapture.InitializeAsync(settings).get();
 
+    if (options.enable) { g_mediaCapture.AddVideoEffectAsync(MRCVideoEffect(options), MediaStreamType::VideoRecord).get(); }
+
     PersonalVideo_FindVideoSource(g_mediaCapture, g_videoSource);
 
     g_ready = true;

hl2ss/hl2ss/personal_video.h

@@ -1,11 +1,12 @@
 
 #pragma once
 
+#include "custom_video_effect.h"
 #include <winrt/Windows.Media.Capture.Frames.h>
 
 void PersonalVideo_Initialize();
 void PersonalVideo_Cleanup();
-void PersonalVideo_Open();
+void PersonalVideo_Open(MRCOptions const& options);
 void PersonalVideo_Close();
 bool PersonalVideo_Status();
 bool PersonalVideo_SetFormat(uint32_t width, uint32_t height, uint32_t framerate);

hl2ss/hl2ss/stream_pv.cpp

@@ -251,12 +251,19 @@ static void PV_Stream(SOCKET clientsocket)
     ok = recv_u8(clientsocket, mode);
     if (!ok) { return; }
 
-    if (!PersonalVideo_Status() && (mode & 4)) { PersonalVideo_Open(); }
-    if (!PersonalVideo_Status()) { return; }
-
     ok = ReceiveH26xFormat_Video(clientsocket, format);
     if (!ok) { return; }
 
+    if (mode & 4)
+    {
+    MRCOptions options;
+    ok = ReceiveMRCOptions(clientsocket, options);
+    if (!ok) { return; }
+    if (!PersonalVideo_Status()) { PersonalVideo_Open(options); }
+    }
+
+    if (!PersonalVideo_Status()) { return; }
+
     ok = PersonalVideo_SetFormat(format.width, format.height, format.framerate);
     if (!ok) { return; }
 

hl2ss/plugin/plugin.vcxproj

@@ -281,6 +281,7 @@
     <ClCompile Include="..\hl2ss\custom_media_type_handler.cpp" />
     <ClCompile Include="..\hl2ss\custom_sink_writers.cpp" />
     <ClCompile Include="..\hl2ss\custom_stream_sink.cpp" />
+    <ClCompile Include="..\hl2ss\custom_video_effect.cpp" />
     <ClCompile Include="..\hl2ss\extended_eye_tracking.cpp" />
     <ClCompile Include="..\hl2ss\ipc_rc.cpp" />
     <ClCompile Include="..\hl2ss\ipc_sc.cpp" />
@@ -352,6 +353,7 @@
     <ClInclude Include="..\hl2ss\custom_sink_hook.h" />
     <ClInclude Include="..\hl2ss\custom_sink_writers.h" />
     <ClInclude Include="..\hl2ss\custom_stream_sink.h" />
+    <ClInclude Include="..\hl2ss\custom_video_effect.h" />
     <ClInclude Include="..\hl2ss\extended_eye_tracking.h" />
     <ClInclude Include="..\hl2ss\ipc_rc.h" />
     <ClInclude Include="..\hl2ss\ipc_sc.h" />

hl2ss/plugin/plugin.vcxproj.filters

@@ -219,6 +219,9 @@
     <ClCompile Include="..\..\3rdparty\Zdepth\src\zdepth.cpp">
       <Filter>Libraries\Zdepth</Filter>
     </ClCompile>
+    <ClCompile Include="..\hl2ss\custom_video_effect.cpp">
+      <Filter>Libraries\hl2ss</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\hl2ss\custom_hook_callback.h">
@@ -434,6 +437,9 @@
     <ClInclude Include="..\..\3rdparty\Zdepth\include\zdepth.hpp">
       <Filter>Libraries\Zdepth</Filter>
     </ClInclude>
+    <ClInclude Include="..\hl2ss\custom_video_effect.h">
+      <Filter>Libraries\hl2ss</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\packages\Microsoft.MixedReality.SceneUnderstanding.1.0.14\runtimes\win10-arm64\native\Microsoft.MixedReality.SceneUnderstanding.dll">

viewer/client_stream_pv.py

@@ -26,6 +26,9 @@
 # 2: query calibration (single transfer)
 mode = hl2ss.StreamMode.MODE_1
 
+# Enable Mixed Reality Capture (Holograms)
+enable_mrc = False
+
 # Camera parameters
 width     = 1920
 height    = 1080
@@ -49,7 +52,7 @@
 
 #------------------------------------------------------------------------------
 
-hl2ss_lnm.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO)
+hl2ss_lnm.start_subsystem_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, enable_mrc=enable_mrc)
 
 if (mode == hl2ss.StreamMode.MODE_2):
     data = hl2ss_lnm.download_calibration_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, width, height, framerate)

viewer/hl2ss.py

@@ -161,6 +161,11 @@ class PNGFilterMode:
     ADAPTIVE  = 6
 
 
+class HologramPerspective:
+    DISPLAY = 0
+    PV      = 1
+
+
 # RM VLC Parameters
 class Parameters_RM_VLC:
     WIDTH  = 640
@@ -440,7 +445,11 @@ def _create_configuration_for_h26x_encoding(options):
     for key, value in options.items():
         configuration.extend(struct.pack('<QQ', key, value))
     return bytes(configuration)
-
+
+
+def _create_configuration_for_mrc(enable, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective):
+    return struct.pack('<BBBBBBfffII', 1 if (enable) else 0, 1 if (hologram_composition) else 0, 1 if (recording_indicator) else 0, 1 if (video_stabilization) else 0, 1 if (blank_protected) else 0, 1 if (show_mesh) else 0, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective)
+
 
 def _create_configuration_for_rm_vlc(mode, divisor, profile, level, bitrate, options):
     configuration = bytearray()
@@ -567,10 +576,11 @@ class _PVCNT:
     MODE_3 = 0x03
 
 
-def start_subsystem_pv(host, port):
+def start_subsystem_pv(host, port, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective):
     c = _client()
     c.open(host, port)
     c.sendall(_create_configuration_for_pv_mode2(_PVCNT.START | _PVCNT.MODE_3, 1920, 1080, 30))
+    c.sendall(_create_configuration_for_mrc(enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective))
     c.close()
 
 

viewer/hl2ss_lnm.py

@@ -70,8 +70,8 @@ def get_sync_period(rx):
 # Control
 #------------------------------------------------------------------------------
 
-def start_subsystem_pv(host, port):
-    hl2ss.start_subsystem_pv(host, port)
+def start_subsystem_pv(host, port, enable_mrc=False, hologram_composition=True, recording_indicator=False, video_stabilization=False, blank_protected=False, show_mesh=False, global_opacity=0.9, output_width=0.0, output_height=0.0, video_stabilization_length=0, hologram_perspective=hl2ss.HologramPerspective.PV):
+    hl2ss.start_subsystem_pv(host, port, enable_mrc, hologram_composition, recording_indicator, video_stabilization, blank_protected, show_mesh, global_opacity, output_width, output_height, video_stabilization_length, hologram_perspective)
 
 
 def stop_subsystem_pv(host, port):