diff --git a/etc/hl2_video_device_controller.txt b/etc/hl2_video_device_controller.txt index b941d7b3a..ed58261e2 100644 --- a/etc/hl2_video_device_controller.txt +++ b/etc/hl2_video_device_controller.txt @@ -83,3 +83,109 @@ Supported Presets 7 8 Value: 0 + +PersonalVideo_SetBrightness +Supported 0 +AutoModeSupported 0 +Min 0 +Max 0 +Default 0 +Step 0 + +PersonalVideo_SetContrast +Supported 0 +AutoModeSupported 0 +Min 0 +Max 0 +Default 0 +Step 0 + +PersonalVideo_ExposureCompensationControl +Supported 1 +Min 0 +Max 0 +Value 0 +Step 1610612736 + +PersonalVideo_Pan +Supported 0 +AutoModeSupported 0 +Min 0 +Max 0 +Default 0 +Step 0 + +PersonalVideo_Roll +Supported 0 +AutoModeSupported 0 +Min 0 +Max 0 +Default 0 +Step 0 + +PersonalVideo_Tilt +Supported 0 +AutoModeSupported 0 +Min 0 +Max 0 +Default 0 +Step 0 + +PersonalVideo_SetDesiredOptimization +Value 0 + +PersonalVideo_SetPrimaryUse +Value 1 + +PersonalVideo_SetZoom +Supported 1 +Supported Modes +0 +1 +2 +Mode 0 +Min 0 +Max 0 +Value 0 +Step 0 + +PersonalVideo_SetOpticalImageStabilization +Supported 1 +Supported Modes +0 +1 +Mode 1 + +PersonalVideo_SetHue +Supported 0 +AutoModeSupported 0 +Min 0 +Max 0 +Default 0 +Step 0 + +PersonalVideo_SetHdrVideo +Supported 1 +Supported Modes +0 +1 +2 +Mode 0 + +PersonalVideo_SetFlashControl +Supported 0 + +PersonalVideo_SetTorchControl +Supported 0 + +PersonalVideo_SetInfraredTorch +Supported 0 + +PersonalVideo_SetPanelBasedOptimization +Supported 0 + +PersonalVideo_SetRegionsOfInterest +AutoExposureSupported 1 +AutoWhiteBalanceSupported 0 +AutoFocusSupported 1 +Max Regions 1 diff --git a/etc/hl2ss_1.0.26.0_arm64_FLAT.appxbundle b/etc/hl2ss_1.0.26.0_arm64_FLAT.appxbundle deleted file mode 100644 index 18e644c8a..000000000 Binary files a/etc/hl2ss_1.0.26.0_arm64_FLAT.appxbundle and /dev/null differ diff --git a/etc/pv_properties.txt b/etc/pv_properties.txt index e92fa45d6..d249cead1 100644 --- a/etc/pv_properties.txt +++ b/etc/pv_properties.txt @@ -11,10 +11,10 @@ {9BEC3FD1-D3FD-4816-8943-DBFEB5B43372} : MFSampleExtension_FrameServer_AllocatorType : Windows.Foundation.IReferenceArray`1 => {24, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0]} {137E6B95-4CAD-4CB9-9B7F-65CEF2068A41} : MFSampleExtension_EOS : Windows.Foundation.IReference`1 => 0 {03FFDCB2-C32C-4B92-98FF-87C0333DB253} : MFSampleExtension_FSMediaTypeInfo : Windows.Foundation.IReferenceArray`1 => {40, [3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 78, 86, 49, 50, 78, 86, 49, 50, 0, 0, 16, 0, 128, 0, 0, 170, 0, 56, 155, 113, 128, 7, 0, 0, 56, 4, 0, 0]} -{6B761658-B7EC-4C3B-8225-8623CABEC31D} : MFSampleExtension_CameraExtrinsics : Windows.Foundation.IReferenceArray`1 => {48, [1, 0, 0, 0, 81, 26, 21, 1, 29, 241, 125, 73, 132, 25, 89, 228, 167, 230, 68, 222, 43, 11, 110, 59, 59, 192, 76, 189, 163, 211, 90, 60, 250, 51, 55, 63, 173, 199, 50, 191, 171, 223, 248, 58, 47, 200, 75, 188]} +{6B761658-B7EC-4C3B-8225-8623CABEC31D} : MFSampleExtension_CameraExtrinsics : Windows.Foundation.IReferenceArray`1 => {48, [1, 0, 0, 0, 81, 26, 21, 1, 29, 241, 125, 73, 132, 25, 89, 228, 167, 230, 68, 222, 43, 11, 110, 59, 59, 192, 76, 189, 163, 211, 90, 60, 250, 51, 55, 63, 173, 199, 50, 191, 171, 223, 248, 58, 47, 200, 75, 188]} : float, 5 garbage? followed by rignode transform translation (y,x,z) and quaternion (x,y,z,w), constant {3680913A-0E20-4B48-9364-BE67FBF1A5E6} : _FrameServerClient_SampleMediaTypeIndex : Windows.Foundation.IReference`1 => 1 {9CDF01D8-A0F0-43BA-B077-EAA06CBD728A} : MFSampleExtension_CleanPoint : Windows.Foundation.IReference`1 => 1 -{4EE3B6C5-6A15-4E72-9761-70C1DB8B9FE3} : MFSampleExtension_PinholeCameraIntrinsics : Windows.Foundation.IReferenceArray`1 => {48, [1, 0, 0, 0, 128, 7, 0, 0, 56, 4, 0, 0, 152, 184, 181, 68, 96, 254, 181, 68, 82, 78, 108, 68, 45, 111, 255, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} +{4EE3B6C5-6A15-4E72-9761-70C1DB8B9FE3} : MFSampleExtension_PinholeCameraIntrinsics : Windows.Foundation.IReferenceArray`1 => {48, [1, 0, 0, 0, 128, 7, 0, 0, 56, 4, 0, 0, 152, 184, 181, 68, 96, 254, 181, 68, 82, 78, 108, 68, 45, 111, 255, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} : float, same as CameraIntrinsics(), 3 garbage? followed by intrinsics (fx, fy, cx, cy) followed by 5 zeros, not constant {B1D5830A-DEB8-40E3-90FA-389943716461} : MFSampleExtension_Interlaced : Windows.Foundation.IReference`1 => 1 {3524E3B0-C355-49F1-B8B0-D0DD62DE37A7} : MFSampleExtension_CameraExtrinsicsCalibrationType : Windows.Foundation.IReference`1 => 1 @@ -55,4 +55,4 @@ {1652C33D-D6B2-4012-B834-72030849A37D} : MF_MT_FRAME_SIZE : Windows.Foundation.IReference`1 => 8246337209400 {B8EBEFAF-B718-4E04-B0A9-116775E3321B} : MF_MT_FIXED_SIZE_SAMPLES : Windows.Foundation.IReference`1 => 1 {6CD8CF3B-6443-48BE-8BC2-01A278E79897} : ? : Windows.Foundation.IReference`1 => 1 -{86B6ADBB-3735-447D-BEE5-6FC23CB58D4A} : ? (intrinisics?) : Windows.Foundation.IReferenceArray`1 => {48, [1, 0, 0, 0, 128, 7, 0, 0, 56, 4, 0, 0, 127, 174, 183, 68, 113, 245, 183, 68, 56, 85, 108, 68, 19, 205, 255, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} +{86B6ADBB-3735-447D-BEE5-6FC23CB58D4A} : ? (intrinsics?) : Windows.Foundation.IReferenceArray`1 => {48, [1, 0, 0, 0, 128, 7, 0, 0, 56, 4, 0, 0, 127, 174, 183, 68, 113, 245, 183, 68, 56, 85, 108, 68, 19, 205, 255, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} float, close but not the same as CameraIntrinsics(), 3 garbage? followed by intrinsics (fx, fy, cx, cy) followed by 5 zeros, constant diff --git a/extensions/client_cpp/hl2ss.cpp b/extensions/client_cpp/hl2ss.cpp index 2b6e29037..bb43ccff6 100644 --- a/extensions/client_cpp/hl2ss.cpp +++ b/extensions/client_cpp/hl2ss.cpp @@ -741,11 +741,13 @@ void decoder_rm_vlc::open(uint8_t profile) std::unique_ptr decoder_rm_vlc::decode(uint8_t* data, uint32_t size) { - std::unique_ptr out = std::make_unique(DECODED_SIZE); + uint32_t const h26x_size = size - METADATA_SIZE; + std::unique_ptr out = std::make_unique(DECODED_SIZE + METADATA_SIZE); if (m_profile != video_profile::RAW) { - std::shared_ptr f = m_codec.decode(data, size); - memcpy(out.get(), f->av_frame->data[0], parameters_rm_vlc::PIXELS); + std::shared_ptr f = m_codec.decode(data, h26x_size); + memcpy(out.get(), f->av_frame->data[0], DECODED_SIZE); + memcpy(out.get() + DECODED_SIZE, data + h26x_size, METADATA_SIZE); } else { @@ -768,18 +770,20 @@ void decoder_rm_depth_ahat::open(uint8_t profile_z, uint8_t profile_ab) std::unique_ptr decoder_rm_depth_ahat::decode(uint8_t* data, uint32_t size) { - int const base = 2 * sizeof(uint32_t); - - std::unique_ptr out = std::make_unique(DECODED_SIZE); + uint32_t const base = 2 * sizeof(uint32_t); + uint32_t const data_size = size - METADATA_SIZE - base; + + std::unique_ptr out = std::make_unique(DECODED_SIZE + METADATA_SIZE); uint32_t sz = DECODED_SIZE / 2; uint8_t* dst_z = out.get(); - uint8_t* dst_ab = out.get() + sz; + uint8_t* dst_ab = dst_z + sz; + uint8_t* dst_md = dst_ab + sz; if (m_profile_z == depth_profile::SAME) { if (m_profile_ab != video_profile::RAW) { - std::shared_ptr f = m_codec.decode(data + base, size); + std::shared_ptr f = m_codec.decode(data + base, data_size); cv::Mat depth = cv::Mat(parameters_rm_depth_ahat::HEIGHT, parameters_rm_depth_ahat::WIDTH, CV_16UC1, dst_z); cv::Mat ab = cv::Mat(parameters_rm_depth_ahat::HEIGHT, parameters_rm_depth_ahat::WIDTH, CV_16UC1, dst_ab); @@ -796,10 +800,12 @@ std::unique_ptr decoder_rm_depth_ahat::decode(uint8_t* data, uint32_t cc.convertTo(bb, CV_16UC1); cv::multiply(bb, bb, bb); cv::resize(bb, ab, ab.size(), 0, 0, cv::INTER_NEAREST); + + memcpy(dst_md, data + base + data_size, METADATA_SIZE); } else { - memcpy(out.get(), data + base, size); + memcpy(out.get(), data + base, size - base); } } else @@ -811,8 +817,9 @@ std::unique_ptr decoder_rm_depth_ahat::decode(uint8_t* data, uint32_t memcpy(&sz_depth, data, sizeof(sz_depth)); memcpy(&sz_ab, data + sizeof(uint32_t), sizeof(sz_ab)); - uint8_t* data_z = data + base; - uint8_t* data_ab = data_z + sz_depth; + uint8_t* data_z = data + base; + uint8_t* data_ab = data_z + sz_depth; + uint8_t* data_md = data_ab + sz_ab; if (sz_depth > 0) { @@ -845,6 +852,8 @@ std::unique_ptr decoder_rm_depth_ahat::decode(uint8_t* data, uint32_t { memcpy(dst_ab, data_ab, sz); } + + memcpy(dst_md, data_md, METADATA_SIZE); #else throw std::runtime_error("hl2ss::decoder_rm_depth_ahat::decode : ZDEPTH decompression not implemented"); #endif @@ -863,9 +872,11 @@ void decoder_rm_depth_longthrow::open() std::unique_ptr decoder_rm_depth_longthrow::decode(uint8_t* data, uint32_t size) { - std::unique_ptr out = std::make_unique(DECODED_SIZE); + uint32_t const png_size = size - METADATA_SIZE; + std::unique_ptr out = std::make_unique(DECODED_SIZE + METADATA_SIZE); cv::Mat output = cv::Mat(parameters_rm_depth_longthrow::HEIGHT, parameters_rm_depth_longthrow::WIDTH, CV_8UC4, out.get()); - cv::imdecode(cv::Mat(1, size, CV_8UC1, data), cv::IMREAD_UNCHANGED, &output); + cv::imdecode(cv::Mat(1, png_size, CV_8UC1, data), cv::IMREAD_UNCHANGED, &output); + memcpy(out.get() + DECODED_SIZE, data + png_size, METADATA_SIZE); return out; } @@ -906,7 +917,7 @@ void decoder_pv::resolution_decoded(uint32_t payload_size, uint8_t decoded_forma { uint16_t stride; channels = decoded_bpp(decoded_format); - resolution((payload_size - K_SIZE) / channels, width, height, stride); + resolution((payload_size - METADATA_SIZE) / channels, width, height, stride); } void decoder_pv::open(uint16_t width, uint16_t height, uint8_t profile) @@ -920,7 +931,7 @@ void decoder_pv::open(uint16_t width, uint16_t height, uint8_t profile) std::unique_ptr decoder_pv::decode(uint8_t* data, uint32_t size, uint8_t decoded_format, uint32_t& decoded_size) { - uint32_t h26x_size = size - K_SIZE; + uint32_t const h26x_size = size - METADATA_SIZE; uint16_t width; uint16_t height; uint16_t stride; @@ -944,11 +955,11 @@ std::unique_ptr decoder_pv::decode(uint8_t* data, uint32_t size, uint code = cv_format[decoded_format][3]; } - decoded_size = width * height * decoded_bpp(decoded_format) + K_SIZE; - std::unique_ptr out = std::make_unique(decoded_size); + decoded_size = width * height * decoded_bpp(decoded_format); + std::unique_ptr out = std::make_unique(decoded_size + METADATA_SIZE); cv::Mat output = cv::Mat(height, width, cv_format[decoded_format][1], out.get()); cv::cvtColor(input, output, code); - memcpy(out.get() + decoded_size - K_SIZE, data + h26x_size, K_SIZE); + memcpy(out.get() + decoded_size, data + h26x_size, METADATA_SIZE); return out; } @@ -1004,7 +1015,7 @@ void rx_decoded_rm_vlc::open() std::shared_ptr rx_decoded_rm_vlc::get_next_packet() { std::shared_ptr p = rx_rm_vlc::get_next_packet(); - if (profile != video_profile::RAW) { p->set_payload(decoder_rm_vlc::DECODED_SIZE, m_decoder.decode(p->payload.get(), p->sz_payload)); } + if (profile != video_profile::RAW) { p->set_payload(decoder_rm_vlc::DECODED_SIZE + decoder_rm_vlc::METADATA_SIZE, m_decoder.decode(p->payload.get(), p->sz_payload)); } return p; } @@ -1027,7 +1038,7 @@ void rx_decoded_rm_depth_ahat::open() std::shared_ptr rx_decoded_rm_depth_ahat::get_next_packet() { std::shared_ptr p = rx_rm_depth_ahat::get_next_packet(); - if ((profile_z != depth_profile::SAME) || (profile_ab != video_profile::RAW)) { p->set_payload(decoder_rm_depth_ahat::DECODED_SIZE, m_decoder.decode(p->payload.get(), p->sz_payload)); } + if ((profile_z != depth_profile::SAME) || (profile_ab != video_profile::RAW)) { p->set_payload(decoder_rm_depth_ahat::DECODED_SIZE + decoder_rm_depth_ahat::METADATA_SIZE, m_decoder.decode(p->payload.get(), p->sz_payload)); } return p; } @@ -1050,7 +1061,7 @@ void rx_decoded_rm_depth_longthrow::open() std::shared_ptr rx_decoded_rm_depth_longthrow::get_next_packet() { std::shared_ptr p = rx_rm_depth_longthrow::get_next_packet(); - p->set_payload(decoder_rm_depth_longthrow::DECODED_SIZE, m_decoder.decode(p->payload.get(), p->sz_payload)); + p->set_payload(decoder_rm_depth_longthrow::DECODED_SIZE + decoder_rm_depth_longthrow::METADATA_SIZE, m_decoder.decode(p->payload.get(), p->sz_payload)); return p; } @@ -1076,7 +1087,7 @@ std::shared_ptr rx_decoded_pv::get_next_packet() std::shared_ptr p = rx_pv::get_next_packet(); uint32_t size; std::unique_ptr payload = m_decoder.decode(p->payload.get(), p->sz_payload, decoded_format, size); - p->set_payload(size, std::move(payload)); + p->set_payload(size + decoder_pv::METADATA_SIZE, std::move(payload)); return p; } @@ -1146,12 +1157,7 @@ std::shared_ptr download_calibration_rm_vlc(char const* host c.open(host, port); c.sendall(sc.data(), sc.size()); - - c.download(data->uv2xy, sizeof(data->uv2xy), chunk_size::SINGLE_TRANSFER); - c.download(data->extrinsics, sizeof(data->extrinsics), chunk_size::SINGLE_TRANSFER); - c.download(data->undistort_map, sizeof(data->undistort_map), chunk_size::SINGLE_TRANSFER); - c.download(data->intrinsics, sizeof(data->intrinsics), chunk_size::SINGLE_TRANSFER); - + c.download(data.get(), sizeof(calibration_rm_vlc), chunk_size::SINGLE_TRANSFER); c.close(); return data; @@ -1167,14 +1173,7 @@ std::shared_ptr download_calibration_rm_depth_ahat(ch c.open(host, port); c.sendall(sc.data(), sc.size()); - - c.download(data->uv2xy, sizeof(data->uv2xy), chunk_size::SINGLE_TRANSFER); - c.download(data->extrinsics, sizeof(data->extrinsics), chunk_size::SINGLE_TRANSFER); - c.download(&data->scale, sizeof(data->scale), chunk_size::SINGLE_TRANSFER); - c.download(&data->alias, sizeof(data->alias), chunk_size::SINGLE_TRANSFER); - c.download(data->undistort_map, sizeof(data->undistort_map), chunk_size::SINGLE_TRANSFER); - c.download(data->intrinsics, sizeof(data->intrinsics), chunk_size::SINGLE_TRANSFER); - + c.download(data.get(), sizeof(calibration_rm_depth_ahat), chunk_size::SINGLE_TRANSFER); c.close(); return data; @@ -1190,13 +1189,7 @@ std::shared_ptr download_calibration_rm_depth_lo c.open(host, port); c.sendall(sc.data(), sc.size()); - - c.download(data->uv2xy, sizeof(data->uv2xy), chunk_size::SINGLE_TRANSFER); - c.download(data->extrinsics, sizeof(data->extrinsics), chunk_size::SINGLE_TRANSFER); - c.download(&data->scale, sizeof(data->scale), chunk_size::SINGLE_TRANSFER); - c.download(data->undistort_map, sizeof(data->undistort_map), chunk_size::SINGLE_TRANSFER); - c.download(data->intrinsics, sizeof(data->intrinsics), chunk_size::SINGLE_TRANSFER); - + c.download(data.get(), sizeof(calibration_rm_depth_longthrow), chunk_size::SINGLE_TRANSFER); c.close(); return data; @@ -1212,9 +1205,7 @@ std::shared_ptr download_calibration_rm_imu(char const* host c.open(host, port); c.sendall(sc.data(), sc.size()); - - c.download(data->extrinsics, sizeof(data->extrinsics), chunk_size::SINGLE_TRANSFER); - + c.download(data.get(), sizeof(calibration_rm_imu), chunk_size::SINGLE_TRANSFER); c.close(); return data; @@ -1230,14 +1221,7 @@ std::shared_ptr download_calibration_pv(char const* host, uint16 c.open(host, port); c.sendall(sc.data(), sc.size()); - - c.download(data->focal_length, sizeof(data->focal_length), chunk_size::SINGLE_TRANSFER); - c.download(data->principal_point, sizeof(data->principal_point), chunk_size::SINGLE_TRANSFER); - c.download(data->radial_distortion, sizeof(data->radial_distortion), chunk_size::SINGLE_TRANSFER); - c.download(data->tangential_distortion, sizeof(data->tangential_distortion), chunk_size::SINGLE_TRANSFER); - c.download(data->projection, sizeof(data->projection), chunk_size::SINGLE_TRANSFER); - c.download(data->extrinsics, sizeof(data->extrinsics), chunk_size::SINGLE_TRANSFER); - + c.download(data.get(), sizeof(calibration_pv), chunk_size::SINGLE_TRANSFER); c.close(); return data; @@ -1307,20 +1291,27 @@ void ipc::close() namespace cmd_ipc_rc { -uint8_t const GET_APPLICATION_VERSION = 0x00; -uint8_t const GET_UTC_OFFSET = 0x01; -uint8_t const SET_HS_MARKER_STATE = 0x02; -uint8_t const GET_PV_SUBSYSTEM_STATUS = 0x03; -uint8_t const SET_PV_FOCUS = 0x04; -uint8_t const SET_PV_VIDEO_TEMPORAL_DENOISING = 0x05; -uint8_t const SET_PV_WHITE_BALANCE_PRESET = 0x06; -uint8_t const SET_PV_WHITE_BALANCE_VALUE = 0x07; -uint8_t const SET_PV_EXPOSURE = 0x08; -uint8_t const SET_PV_EXPOSURE_PRIORITY_VIDEO = 0x09; -uint8_t const SET_PV_ISO_SPEED = 0x0A; -uint8_t const SET_PV_BACKLIGHT_COMPENSATION = 0x0B; -uint8_t const SET_PV_SCENE_MODE = 0x0C; -uint8_t const SET_FLAT_MODE = 0x0D; +uint8_t const GET_APPLICATION_VERSION = 0x00; +uint8_t const GET_UTC_OFFSET = 0x01; +uint8_t const SET_HS_MARKER_STATE = 0x02; +uint8_t const GET_PV_SUBSYSTEM_STATUS = 0x03; +uint8_t const SET_PV_FOCUS = 0x04; +uint8_t const SET_PV_VIDEO_TEMPORAL_DENOISING = 0x05; +uint8_t const SET_PV_WHITE_BALANCE_PRESET = 0x06; +uint8_t const SET_PV_WHITE_BALANCE_VALUE = 0x07; +uint8_t const SET_PV_EXPOSURE = 0x08; +uint8_t const SET_PV_EXPOSURE_PRIORITY_VIDEO = 0x09; +uint8_t const SET_PV_ISO_SPEED = 0x0A; +uint8_t const SET_PV_BACKLIGHT_COMPENSATION = 0x0B; +uint8_t const SET_PV_SCENE_MODE = 0x0C; +uint8_t const SET_FLAT_MODE = 0x0D; +uint8_t const SET_RM_EYE_SELECTION = 0x0E; +uint8_t const SET_PV_DESIRED_OPTIMIZATION = 0x0F; +uint8_t const SET_PV_PRIMARY_USE = 0x10; +uint8_t const SET_PV_OPTICAL_IMAGE_STABILIZATION = 0x11; +uint8_t const SET_PV_HDR_VIDEO = 0x12; +uint8_t const SET_PV_REGIONS_OF_INTEREST = 0x13; +uint8_t const SET_INTERFACE_PRIORITY = 0x14; }; ipc_rc::ipc_rc(char const* host, uint16_t port) : ipc(host, port) @@ -1424,6 +1415,42 @@ void ipc_rc::set_flat_mode(uint32_t mode) send(cmd_ipc_rc::SET_FLAT_MODE, {mode}); } +void ipc_rc::set_rm_eye_selection(uint32_t enable) +{ + send(cmd_ipc_rc::SET_RM_EYE_SELECTION, {enable}); +} + +void ipc_rc::set_pv_desired_optimization(uint32_t mode) +{ + send(cmd_ipc_rc::SET_PV_DESIRED_OPTIMIZATION, {mode}); +} + +void ipc_rc::set_pv_primary_use(uint32_t mode) +{ + send(cmd_ipc_rc::SET_PV_PRIMARY_USE, {mode}); +} + +void ipc_rc::set_pv_optical_image_stabilization(uint32_t mode) +{ + send(cmd_ipc_rc::SET_PV_OPTICAL_IMAGE_STABILIZATION, {mode}); +} + +void ipc_rc::set_pv_hdr_video(uint32_t mode) +{ + send(cmd_ipc_rc::SET_PV_HDR_VIDEO, {mode}); +} + +void ipc_rc::set_pv_regions_of_interest(bool clear, bool set, bool auto_exposure, bool auto_focus, bool bounds_normalized, uint32_t type, uint32_t weight, float x, float y, float w, float h) +{ + uint32_t mode = (0x1000 * clear) | (0x0800 * set) | (0x0400 * auto_exposure) | (0x0200 * auto_focus) | (0x0100 * bounds_normalized) | ((type & 1) << 7) | (weight & 0x7F); + send(cmd_ipc_rc::SET_PV_REGIONS_OF_INTEREST, {mode, *(uint32_t*)&x, *(uint32_t*)&y, *(uint32_t*)&w, *(uint32_t*)&h}); +} + +void ipc_rc::set_interface_priority(uint16_t port, int32_t priority) +{ + send(cmd_ipc_rc::SET_INTERFACE_PRIORITY, { (uint32_t)port, *(uint32_t*)&priority }); +} + //------------------------------------------------------------------------------ // * Spatial Mapping //------------------------------------------------------------------------------ @@ -1441,41 +1468,53 @@ sm_bounding_volume::sm_bounding_volume() m_count = 0; } -void sm_bounding_volume::add_box(vector_3 center, vector_3 extents) +sm_bounding_volume::sm_bounding_volume(uint32_t count, uint8_t const* data, size_t size) +{ + m_count = count; + m_data = { data, data + size }; +} + +void sm_bounding_volume::add_box(sm_box box) { m_count++; push_u32(m_data, sm_volume_type::Box); - push(m_data, ¢er, sizeof(center)); - push(m_data, &extents, sizeof(extents)); + push(m_data, &box, sizeof(box)); } -void sm_bounding_volume::add_frustum(plane p_near, plane p_far, plane p_right, plane p_left, plane p_top, plane p_bottom) +void sm_bounding_volume::add_frustum(sm_frustum frustum) { m_count++; push_u32(m_data, sm_volume_type::Frustum); - push(m_data, &p_near, sizeof(p_near)); - push(m_data, &p_far, sizeof(p_far)); - push(m_data, &p_right, sizeof(p_right)); - push(m_data, &p_left, sizeof(p_left)); - push(m_data, &p_top, sizeof(p_top)); - push(m_data, &p_bottom, sizeof(p_bottom)); + push(m_data, &frustum, sizeof(frustum)); } -void sm_bounding_volume::add_oriented_box(vector_3 center, vector_3 extents, quaternion orientation) +void sm_bounding_volume::add_oriented_box(sm_oriented_box oriented_box) { m_count++; push_u32(m_data, sm_volume_type::OrientedBox); - push(m_data, ¢er, sizeof(center)); - push(m_data, &extents, sizeof(extents)); - push(m_data, &orientation, sizeof(orientation)); + push(m_data, &oriented_box, sizeof(oriented_box)); } -void sm_bounding_volume::add_sphere(vector_3 center, float radius) +void sm_bounding_volume::add_sphere(sm_sphere sphere) { m_count++; push_u32(m_data, sm_volume_type::Sphere); - push(m_data, ¢er, sizeof(center)); - push_float(m_data, radius); + push(m_data, &sphere, sizeof(sphere)); +} + +uint32_t sm_bounding_volume::get_count() const +{ + return m_count; +} + +uint8_t const* sm_bounding_volume::get_data() const +{ + return m_data.data(); +} + +size_t sm_bounding_volume::get_size() const +{ + return m_data.size(); } sm_mesh_task::sm_mesh_task() @@ -1483,6 +1522,12 @@ sm_mesh_task::sm_mesh_task() m_count = 0; } +sm_mesh_task::sm_mesh_task(uint32_t count, uint8_t const* data, size_t size) +{ + m_count = count; + m_data = { data, data + size }; +} + void sm_mesh_task::add_task(guid id, double max_triangles_per_cubic_meter, uint32_t vertex_position_format, uint32_t triangle_index_format, uint32_t vertex_normal_format, bool include_vertex_normals, bool include_bounds) { m_count++; @@ -1495,6 +1540,21 @@ void sm_mesh_task::add_task(guid id, double max_triangles_per_cubic_meter, uint3 push_u32(m_data, (1*include_vertex_normals) | (2*include_bounds)); } +uint32_t sm_mesh_task::get_count() const +{ + return m_count; +} + +uint8_t const* sm_mesh_task::get_data() const +{ + return m_data.data(); +} + +size_t sm_mesh_task::get_size() const +{ + return m_data.size(); +} + ipc_sm::ipc_sm(char const* host, uint16_t port) : ipc(host, port) { } @@ -1509,8 +1569,8 @@ void ipc_sm::set_volumes(sm_bounding_volume const& volumes) { std::vector sc; push_u8(sc, commmand_ipc_sm::SET_VOLUMES); - push_u8(sc, (uint8_t)volumes.m_count); - push(sc, volumes.m_data.data(), volumes.m_data.size()); + push_u8(sc, (uint8_t)volumes.get_count()); + push(sc, volumes.get_data(), volumes.get_size()); m_client.sendall(sc.data(), sc.size()); } @@ -1528,14 +1588,14 @@ void ipc_sm::get_meshes(sm_mesh_task const& tasks, uint32_t threads, std::vector { std::vector sc; push_u8(sc, commmand_ipc_sm::GET_MESHES); - push_u32(sc, tasks.m_count); + push_u32(sc, tasks.get_count()); push_u32(sc, threads); - push(sc, tasks.m_data.data(), tasks.m_data.size()); + push(sc, tasks.get_data(), tasks.get_size()); m_client.sendall(sc.data(), sc.size()); - meshes.resize(tasks.m_count); + meshes.resize(tasks.get_count()); - for (uint32_t i = 0; i < tasks.m_count; ++i) + for (uint32_t i = 0; i < tasks.get_count(); ++i) { uint32_t index; @@ -1812,21 +1872,24 @@ void ipc_gmq::push(uint32_t const* response, uint32_t count) // * Unpacking //------------------------------------------------------------------------------ -void unpack_rm_vlc(uint8_t* payload, uint8_t*& image) +void unpack_rm_vlc(uint8_t* payload, uint8_t*& image, rm_vlc_metadata*& metadata) { - image = payload; + image = payload; + metadata = (rm_vlc_metadata*)(payload + parameters_rm_vlc::PIXELS); } -void unpack_rm_depth_ahat(uint8_t* payload, uint16_t*& depth, uint16_t*& ab) +void unpack_rm_depth_ahat(uint8_t* payload, uint16_t*& depth, uint16_t*& ab, rm_depth_ahat_metadata*& metadata) { - depth = (uint16_t*)(payload); - ab = (uint16_t*)(payload + (parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t))); + depth = (uint16_t*) (payload); + ab = (uint16_t*) (payload + ( parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t))); + metadata = (rm_depth_ahat_metadata*)(payload + (2 * parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t))); } -void unpack_rm_depth_longthrow(uint8_t* payload, uint16_t*& depth, uint16_t*& ab) +void unpack_rm_depth_longthrow(uint8_t* payload, uint16_t*& depth, uint16_t*& ab, rm_depth_longthrow_metadata*& metadata) { - depth = (uint16_t*)(payload); - ab = (uint16_t*)(payload + (parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t))); + depth = (uint16_t*) (payload); + ab = (uint16_t*) (payload + ( parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t))); + metadata = (rm_depth_longthrow_metadata*)(payload + (2 * parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t))); } void unpack_rm_imu(uint8_t* payload, rm_imu_sample*& samples) @@ -1834,10 +1897,10 @@ void unpack_rm_imu(uint8_t* payload, rm_imu_sample*& samples) samples = (rm_imu_sample*)payload; } -void unpack_pv(uint8_t* payload, size_t size, uint8_t*& image, pv_intrinsics*& intrinsics) +void unpack_pv(uint8_t* payload, size_t size, uint8_t*& image, pv_metadata*& metadata) { - image = payload; - intrinsics = (pv_intrinsics*)(payload + size - sizeof(pv_intrinsics)); + image = payload; + metadata = (pv_metadata*)(payload + size - sizeof(pv_metadata)); } void unpack_microphone_raw(uint8_t* payload, int16_t*& samples) diff --git a/extensions/client_cpp/hl2ss.h b/extensions/client_cpp/hl2ss.h index 8d04faaee..5659e38d2 100644 --- a/extensions/client_cpp/hl2ss.h +++ b/extensions/client_cpp/hl2ss.h @@ -313,6 +313,11 @@ struct ray vector_3 direction; }; +struct uint64x2 +{ + uint64_t val[2]; +}; + //------------------------------------------------------------------------------ // * Client //------------------------------------------------------------------------------ @@ -389,14 +394,6 @@ uint32_t extended_audio_device_mixer_mode(uint32_t mixer_mode, uint32_t device); // * PV Control //------------------------------------------------------------------------------ -struct pv_intrinsics -{ - float fx; - float fy; - float cx; - float cy; -}; - void start_subsystem_pv(char const* host, uint16_t port, bool enable_mrc, bool hologram_composition, bool recording_indicator, bool video_stabilization, bool blank_protected, bool show_mesh, bool shared, float global_opacity, float output_width, float output_height, uint32_t video_stabilization_length, uint32_t hologram_perspective); void stop_subsystem_pv(char const* host, uint16_t port); @@ -586,6 +583,39 @@ class codec // * Decoders //------------------------------------------------------------------------------ +struct rm_vlc_metadata +{ + uint64_t sensor_ticks; + uint64_t exposure; + uint32_t gain; + uint32_t _reserved; +}; + +struct rm_depth_ahat_metadata +{ + uint64_t sensor_ticks; +}; + +struct rm_depth_longthrow_metadata +{ + uint64_t sensor_ticks; +}; + +struct pv_metadata +{ + vector_2 f; + vector_2 c; + uint64_t exposure_time; + uint64x2 exposure_compensation; + uint32_t lens_position; + uint32_t focus_state; + uint32_t iso_speed; + uint32_t white_balance; + vector_2 iso_gains; + vector_3 white_balance_gains; + uint32_t _reserved; +}; + class decoder_rm_vlc { private: @@ -594,6 +624,7 @@ class decoder_rm_vlc public: static uint32_t const DECODED_SIZE = parameters_rm_vlc::PIXELS * sizeof(uint8_t); + static uint32_t const METADATA_SIZE = sizeof(rm_vlc_metadata); void open(uint8_t profile); std::unique_ptr decode(uint8_t* data, uint32_t size); @@ -612,6 +643,7 @@ class decoder_rm_depth_ahat public: static uint32_t const DECODED_SIZE = 2 * parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t); + static uint32_t const METADATA_SIZE = sizeof(rm_depth_ahat_metadata); void open(uint8_t profile_z, uint8_t profile_ab); std::unique_ptr decode(uint8_t* data, uint32_t size); @@ -622,6 +654,7 @@ class decoder_rm_depth_longthrow { public: static uint32_t const DECODED_SIZE = 2 * parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t); + static uint32_t const METADATA_SIZE = sizeof(rm_depth_longthrow_metadata); void open(); std::unique_ptr decode(uint8_t* data, uint32_t size); @@ -637,7 +670,7 @@ class decoder_pv uint8_t m_profile; public: - static uint32_t const K_SIZE = sizeof(pv_intrinsics); + static uint32_t const METADATA_SIZE = sizeof(pv_metadata); static uint8_t decoded_bpp(uint8_t decoded_format); static void resolution(uint32_t bytes, uint16_t& width, uint16_t& height, uint16_t& stride); @@ -790,6 +823,8 @@ struct calibration_pv float tangential_distortion[2]; float projection[4][4]; float extrinsics[4][4]; + float intrinsics_mf[4]; + float extrinsics_mf[7]; }; std::shared_ptr download_calibration_rm_vlc(char const* host, uint16_t port); @@ -922,6 +957,12 @@ uint32_t const Min = 100; uint32_t const Max = 3200; } +namespace pv_backlight_compensation_state +{ +uint32_t const Disable = 0; +uint32_t const Enable = 1; +} + namespace pv_capture_scene_mode { uint32_t const Auto = 0; @@ -938,10 +979,50 @@ uint32_t const NightPortrait = 11; uint32_t const Backlit = 12; } -namespace pv_backlight_compensation_state +namespace pv_media_capture_optimization { -uint32_t const Disable = 0; -uint32_t const Enable = 1; +uint32_t const Default = 0; +uint32_t const Quality = 1; +uint32_t const Latency = 2; +uint32_t const Power = 3; +uint32_t const LatencyThenQuality = 4; +uint32_t const LatencyThenPower = 5; +uint32_t const PowerAndQuality = 6; +} + +namespace pv_capture_use +{ +uint32_t const NotSet = 0; +uint32_t const Photo = 1; +uint32_t const Video = 2; +} + +namespace pv_optical_image_stabilization_mode +{ +uint32_t const Off = 0; +uint32_t const On = 1; +} + +namespace pv_hdr_video_mode +{ +uint32_t const Off = 0; +uint32_t const On = 1; +uint32_t const Auto = 2; +} + +namespace pv_region_of_interest_type +{ +uint32_t const Unknown = 0; +uint32_t const Face = 1; +} + +namespace interface_priority +{ +int32_t const LOWEST = -2; +int32_t const BELOW_NORMAL = -1; +int32_t const NORMAL = 0; +int32_t const ABOVE_NORMAL = 1; +int32_t const HIGHEST = 2; } struct version @@ -975,6 +1056,13 @@ class ipc_rc : public ipc void set_pv_backlight_compensation(uint32_t state); void set_pv_scene_mode(uint32_t mode); void set_flat_mode(uint32_t mode); + void set_rm_eye_selection(uint32_t enable); + void set_pv_desired_optimization(uint32_t mode); + void set_pv_primary_use(uint32_t mode); + void set_pv_optical_image_stabilization(uint32_t mode); + void set_pv_hdr_video(uint32_t mode); + void set_pv_regions_of_interest(bool clear, bool set, bool auto_exposure, bool auto_focus, bool bounds_normalized, uint32_t type, uint32_t weight, float x, float y, float w, float h); + void set_interface_priority(uint16_t port, int32_t priority); }; //------------------------------------------------------------------------------ @@ -1007,21 +1095,53 @@ uint32_t const OrientedBox = 2; uint32_t const Sphere = 3; } -class sm_bounding_volume +struct sm_box { - friend class ipc_sm; + vector_3 center; + vector_3 extents; +}; + +struct sm_frustum +{ + plane p_near; + plane p_far; + plane p_right; + plane p_left; + plane p_top; + plane p_bottom; +}; +struct sm_oriented_box +{ + vector_3 center; + vector_3 extents; + quaternion orientation; +}; + +struct sm_sphere +{ + vector_3 center; + float radius; +}; + +class sm_bounding_volume +{ private: std::vector m_data; uint32_t m_count; public: sm_bounding_volume(); + sm_bounding_volume(uint32_t count, uint8_t const* data, size_t size); - void add_box(vector_3 center, vector_3 extents); - void add_frustum(plane p_near, plane p_far, plane p_right, plane p_left, plane p_top, plane p_bottom); - void add_oriented_box(vector_3 center, vector_3 extents, quaternion orientation); - void add_sphere(vector_3 center, float radius); + void add_box(sm_box box); + void add_frustum(sm_frustum frustum); + void add_oriented_box(sm_oriented_box oriented_box); + void add_sphere(sm_sphere sphere); + + uint32_t get_count() const; + uint8_t const* get_data() const; + size_t get_size() const; }; struct guid @@ -1038,16 +1158,19 @@ struct sm_surface_info class sm_mesh_task { - friend class ipc_sm; - private: std::vector m_data; uint32_t m_count; public: sm_mesh_task(); + sm_mesh_task(uint32_t count, uint8_t const* data, size_t size); void add_task(guid id, double max_triangles_per_cubic_meter, uint32_t vertex_position_format, uint32_t triangle_index_format, uint32_t vertex_normal_format, bool include_vertex_normals, bool include_bounds); + + uint32_t get_count() const; + uint8_t const* get_data() const; + size_t get_size() const; }; struct sm_mesh @@ -1268,6 +1391,15 @@ struct rm_imu_sample float temperature; }; +namespace pv_focus_state +{ +uint32_t const UNINITIALIZED = 0; +uint32_t const LOST = 1; +uint32_t const SEARCHING = 2; +uint32_t const FOCUSED = 3; +uint32_t const FAILED = 4; +} + namespace si_valid { uint32_t const HEAD = 0x01; @@ -1354,11 +1486,11 @@ struct eet_frame uint32_t valid; }; -void unpack_rm_vlc(uint8_t* payload, uint8_t*& image); -void unpack_rm_depth_ahat(uint8_t* payload, uint16_t*& depth, uint16_t*& ab); -void unpack_rm_depth_longthrow(uint8_t* payload, uint16_t*& depth, uint16_t*& ab); +void unpack_rm_vlc(uint8_t* payload, uint8_t*& image, rm_vlc_metadata*& metadata); +void unpack_rm_depth_ahat(uint8_t* payload, uint16_t*& depth, uint16_t*& ab, rm_depth_ahat_metadata*& metadata); +void unpack_rm_depth_longthrow(uint8_t* payload, uint16_t*& depth, uint16_t*& ab, rm_depth_longthrow_metadata*& metadata); void unpack_rm_imu(uint8_t* payload, rm_imu_sample*& samples); -void unpack_pv(uint8_t* payload, size_t size, uint8_t*& image, pv_intrinsics*& intrinsics); +void unpack_pv(uint8_t* payload, size_t size, uint8_t*& image, pv_metadata*& metadata); void unpack_microphone_raw(uint8_t* payload, int16_t*& samples); void unpack_microphone_aac(uint8_t* payload, float*& samples); void unpack_si(uint8_t* payload, si_frame*& si); diff --git a/extensions/client_cpp/hl2ss_ulm.cpp b/extensions/client_cpp/hl2ss_ulm.cpp index 555e48e32..09a595896 100644 --- a/extensions/client_cpp/hl2ss_ulm.cpp +++ b/extensions/client_cpp/hl2ss_ulm.cpp @@ -1,6 +1,7 @@ #include #include +#include "hl2ss.h" #include "hl2ss_lnm.h" #include "hl2ss_mt.h" @@ -24,13 +25,82 @@ namespace ulm struct packet { + int64_t frame_stamp; uint64_t timestamp; uint32_t sz_payload; - uint32_t _reserved; + int32_t status; uint8_t* payload; matrix_4x4* pose; }; +struct sm_mesh +{ + uint32_t status; + vector_3 vertex_position_scale; + uint64_t bounds_size; + uint64_t vertex_positions_size; + uint64_t triangle_indices_size; + uint64_t vertex_normals_size; + matrix_4x4* pose; + uint8_t* bounds_data; + uint8_t* vertex_positions_data; + uint8_t* triangle_indices_data; + uint8_t* vertex_normals_data; +}; + +struct su_mesh +{ + uint64_t vertex_positions_size; + uint64_t triangle_indices_size; + uint8_t* vertex_positions_data; + uint8_t* triangle_indices_data; +}; + +struct su_item +{ + guid id; + int32_t kind; + uint32_t _reserved; + quaternion orientation; + vector_3 position; + int32_t alignment; + vector_2 extents; + uint32_t meshes_count; + uint32_t collider_meshes_count; + matrix_4x4* location; + void* meshes; + void* collider_meshes; +}; + +struct su_result_header +{ + uint32_t status; + uint32_t count; + matrix_4x4* extrinsics; + matrix_4x4* pose; +}; + +struct su_task +{ + bool enable_quads; + bool enable_meshes; + bool enable_only_observed; + bool enable_world_mesh; + uint32_t mesh_lod; + float query_radius; + uint8_t create_mode; + uint8_t kind_flags; + bool get_orientation; + bool get_position; + bool get_location_matrix; + bool get_quad; + bool get_meshes; + bool get_collider_meshes; + uint32_t _reserved; + uint64_t guid_list_size; + guid* guid_list_data; +}; + struct gmq_message { uint32_t command; @@ -244,28 +314,32 @@ HL2SS_ULM_END(void()) // Grab //----------------------------------------------------------------------------- -static void unpack_frame(std::shared_ptr data, void*& frame, hl2ss::ulm::packet& packet) +static void* unpack_frame(std::shared_ptr data, int64_t frame_stamp, int32_t status, hl2ss::ulm::packet& packet) { if (data) { - frame = new std::shared_ptr(data); - packet.timestamp = data->timestamp; - packet.sz_payload = data->sz_payload; - packet.payload = data->payload.get(); - packet.pose = data->pose.get(); + packet.frame_stamp = frame_stamp; + packet.timestamp = data->timestamp; + packet.sz_payload = data->sz_payload; + packet.status = status; + packet.payload = data->payload.get(); + packet.pose = data->pose.get(); } else { - frame = nullptr; - packet.timestamp = 0ULL; - packet.sz_payload = 0UL; - packet.payload = nullptr; - packet.pose = nullptr; + packet.frame_stamp = frame_stamp; + packet.timestamp = 0ULL; + packet.sz_payload = 0UL; + packet.status = status; + packet.payload = nullptr; + packet.pose = nullptr; } + + return new std::shared_ptr(data); // delete } HL2SS_CLIENT_EXPORT -int32_t get_by_index(void* source, int64_t& frame_stamp, int32_t& status, void*& frame, hl2ss::ulm::packet& packet) +void* get_by_index(void* source, int64_t frame_stamp, hl2ss::ulm::packet& packet) HL2SS_ULM_BEGIN { hl2ss::mt::source* s = (hl2ss::mt::source*)source; @@ -273,15 +347,14 @@ HL2SS_ULM_BEGIN std::exception source_error; if (!s->status(source_error)) { throw source_error; } + int32_t status; std::shared_ptr data = s->get_packet(frame_stamp, status); - if (status == 0) { unpack_frame(data, frame, packet); } - - return 0; + return unpack_frame(data, frame_stamp, status, packet); } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) HL2SS_CLIENT_EXPORT -int32_t get_by_timestamp(void* source, uint64_t timestamp, int32_t time_preference, int32_t tiebreak_right, int64_t& frame_stamp, int32_t& status, void*& frame, hl2ss::ulm::packet& packet) +void* get_by_timestamp(void* source, uint64_t timestamp, int32_t time_preference, int32_t tiebreak_right, hl2ss::ulm::packet& packet) HL2SS_ULM_BEGIN { hl2ss::mt::source* s = (hl2ss::mt::source*)source; @@ -289,18 +362,18 @@ HL2SS_ULM_BEGIN std::exception source_error; if (!s->status(source_error)) { throw source_error; } + int64_t frame_stamp; + int32_t status; std::shared_ptr data = s->get_packet(timestamp, time_preference, tiebreak_right, frame_stamp, status); - if (status == 0) { unpack_frame(data, frame, packet); } - - return 0; + return unpack_frame(data, frame_stamp, status, packet); } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) HL2SS_CLIENT_EXPORT -void release_frame(void* frame) +void release_packet(void* reference) HL2SS_ULM_BEGIN { - delete (std::shared_ptr*)frame; + delete (std::shared_ptr*)reference; } HL2SS_ULM_END(void()) @@ -331,49 +404,94 @@ HL2SS_ULM_END(-1) //----------------------------------------------------------------------------- HL2SS_CLIENT_EXPORT -int32_t download_calibration_rm_vlc(char const* host, uint16_t port, calibration_rm_vlc& calibration) +void* download_calibration_rm_vlc(char const* host, uint16_t port, hl2ss::calibration_rm_vlc*& calibration) HL2SS_ULM_BEGIN { - calibration = *hl2ss::lnm::download_calibration_rm_vlc(host, port); - return 0; + std::shared_ptr data = hl2ss::lnm::download_calibration_rm_vlc(host, port); + calibration = data.get(); + return new std::shared_ptr(data); // delete } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) HL2SS_CLIENT_EXPORT -int32_t download_calibration_rm_depth_ahat(char const* host, uint16_t port, calibration_rm_depth_ahat& calibration) +void* download_calibration_rm_depth_ahat(char const* host, uint16_t port, hl2ss::calibration_rm_depth_ahat*& calibration) HL2SS_ULM_BEGIN { - calibration = *hl2ss::lnm::download_calibration_rm_depth_ahat(host, port); - return 0; + std::shared_ptr data = hl2ss::lnm::download_calibration_rm_depth_ahat(host, port); + calibration = data.get(); + return new std::shared_ptr(data); // delete } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) HL2SS_CLIENT_EXPORT -int32_t download_calibration_rm_depth_longthrow(char const* host, uint16_t port, calibration_rm_depth_longthrow& calibration) +void* download_calibration_rm_depth_longthrow(char const* host, uint16_t port, calibration_rm_depth_longthrow*& calibration) HL2SS_ULM_BEGIN { - calibration = *hl2ss::lnm::download_calibration_rm_depth_longthrow(host, port); - return 0; + std::shared_ptr data = hl2ss::lnm::download_calibration_rm_depth_longthrow(host, port); + calibration = data.get(); + return new std::shared_ptr(data); // delete } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) HL2SS_CLIENT_EXPORT -int32_t download_calibration_rm_imu(char const* host, uint16_t port, calibration_rm_imu& calibration) +void* download_calibration_rm_imu(char const* host, uint16_t port, calibration_rm_imu*& calibration) HL2SS_ULM_BEGIN { - calibration = *hl2ss::lnm::download_calibration_rm_imu(host, port); - return 0; + std::shared_ptr data = hl2ss::lnm::download_calibration_rm_imu(host, port); + calibration = data.get(); + return new std::shared_ptr(data); // delete } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) HL2SS_CLIENT_EXPORT -int32_t download_calibration_pv(char const* host, uint16_t port, uint16_t width, uint16_t height, uint8_t framerate, calibration_pv& calibration) +void* download_calibration_pv(char const* host, uint16_t port, uint16_t width, uint16_t height, uint8_t framerate, calibration_pv*& calibration) HL2SS_ULM_BEGIN { - calibration = *hl2ss::lnm::download_calibration_pv(host, port, width, height, framerate); - return 0; + std::shared_ptr data = hl2ss::lnm::download_calibration_pv(host, port, width, height, framerate); + calibration = data.get(); + return new std::shared_ptr(data); // delete } -HL2SS_ULM_END(-1) +HL2SS_ULM_END(nullptr) + +HL2SS_CLIENT_EXPORT +void release_calibration_rm_vlc(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::shared_ptr*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +void release_calibration_rm_depth_ahat(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::shared_ptr*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +void release_calibration_rm_depth_longthrow(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::shared_ptr*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +void release_calibration_rm_imu(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::shared_ptr*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +void release_calibration_pv(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::shared_ptr*)reference; +} +HL2SS_ULM_END(void()) //------------------------------------------------------------------------------ // Remote Configuration @@ -514,17 +632,240 @@ HL2SS_ULM_BEGIN } HL2SS_ULM_END(-1) +HL2SS_CLIENT_EXPORT +int32_t rc_set_rm_eye_selection(void* ipc, uint32_t enable) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_rm_eye_selection(enable); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t rc_set_pv_desired_optimization(void* ipc, uint32_t mode) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_pv_desired_optimization(mode); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t rc_set_pv_primary_use(void* ipc, uint32_t mode) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_pv_primary_use(mode); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t rc_set_pv_optical_image_stabilization(void* ipc, uint32_t mode) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_pv_optical_image_stabilization(mode); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t rc_set_pv_hdr_video(void* ipc, uint32_t mode) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_pv_hdr_video(mode); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t rc_set_pv_regions_of_interest(void* ipc, uint32_t clear, uint32_t set, uint32_t auto_exposure, uint32_t auto_focus, uint32_t bounds_normalized, uint32_t type, uint32_t weight, float x, float y, float w, float h) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_pv_regions_of_interest(clear, set, auto_exposure, auto_focus, bounds_normalized, type, weight, x, y, w, h); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t rc_set_interface_priority(void* ipc, uint16_t port, int32_t priority) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_rc*)ipc)->set_interface_priority(port, priority); + return 0; +} +HL2SS_ULM_END(-1) + //------------------------------------------------------------------------------ // Spatial Mapping //------------------------------------------------------------------------------ -// TODO: +HL2SS_CLIENT_EXPORT +int32_t sm_create_observer(void* ipc) +HL2SS_ULM_BEGIN +{ + ((hl2ss::ipc_sm*)ipc)->create_observer(); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t sm_set_volumes(void* ipc, uint32_t count, uint8_t const* data, uint64_t size) +HL2SS_ULM_BEGIN +{ + hl2ss::sm_bounding_volume volumes(count, data, size); + ((hl2ss::ipc_sm*)ipc)->set_volumes(volumes); + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +void* sm_get_observed_surfaces(void* ipc, uint64_t& size, hl2ss::sm_surface_info*& data) +HL2SS_ULM_BEGIN +{ + std::unique_ptr> surfaces = std::make_unique>(); + ((hl2ss::ipc_sm*)ipc)->get_observed_surfaces(*surfaces); + + size = surfaces->size(); + data = surfaces->data(); + + return surfaces.release(); +} +HL2SS_ULM_END(nullptr) + +HL2SS_CLIENT_EXPORT +void sm_release_surfaces(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::vector*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +void* sm_get_meshes(void* ipc, uint32_t count, uint8_t const* data, uint64_t size, uint32_t threads) +HL2SS_ULM_BEGIN +{ + hl2ss::sm_mesh_task tasks(count, data, size); + std::unique_ptr> meshes = std::make_unique>(); + ((hl2ss::ipc_sm*)ipc)->get_meshes(tasks, threads, *meshes); + return meshes.release(); +} +HL2SS_ULM_END(nullptr) + +HL2SS_CLIENT_EXPORT +void sm_release_meshes(void* reference) +HL2SS_ULM_BEGIN +{ + delete (std::vector*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +int32_t sm_unpack_mesh(void* reference, uint32_t index, hl2ss::ulm::sm_mesh& mesh) +HL2SS_ULM_BEGIN +{ + std::vector& meshes = *(std::vector*)reference; + hl2ss::sm_mesh& m = meshes[index]; + + mesh.status = m.status; + mesh.vertex_position_scale = m.vertex_position_scale; + mesh.pose = &m.pose; + mesh.bounds_data = m.bounds.data(); + mesh.bounds_size = m.bounds.size(); + mesh.vertex_positions_data = m.vertex_positions.data(); + mesh.vertex_positions_size = m.vertex_positions.size(); + mesh.triangle_indices_data = m.triangle_indices.data(); + mesh.triangle_indices_size = m.triangle_indices.size(); + mesh.vertex_normals_data = m.vertex_normals.data(); + mesh.vertex_normals_size = m.vertex_normals.size(); + + return 0; +} +HL2SS_ULM_END(-1) //------------------------------------------------------------------------------ // Scene Understanding //------------------------------------------------------------------------------ -// TODO: +HL2SS_CLIENT_EXPORT +void* su_query(void* ipc, hl2ss::ulm::su_task const& task, hl2ss::ulm::su_result_header& header) +HL2SS_ULM_BEGIN +{ + hl2ss::su_task t; + + t.enable_quads = task.enable_quads; + t.enable_meshes = task.enable_meshes; + t.enable_only_observed = task.enable_only_observed; + t.enable_world_mesh = task.enable_world_mesh; + t.mesh_lod = task.mesh_lod; + t.query_radius = task.query_radius; + t.create_mode = task.create_mode; + t.kind_flags = task.kind_flags; + t.get_orientation = task.get_orientation; + t.get_position = task.get_position; + t.get_location_matrix = task.get_location_matrix; + t.get_quad = task.get_quad; + t.get_meshes = task.get_meshes; + t.get_collider_meshes = task.get_collider_meshes; + t.guid_list = { task.guid_list_data, task.guid_list_data + task.guid_list_size }; + + std::unique_ptr result = std::make_unique(); + ((hl2ss::ipc_su*)ipc)->query(t, *result); + + header.status = result->status; + header.extrinsics = &result->extrinsics; + header.pose = &result->pose; + header.count = (uint32_t)result->items.size(); + + return result.release(); +} +HL2SS_ULM_END(nullptr) + +HL2SS_CLIENT_EXPORT +void su_release(void* reference) +HL2SS_ULM_BEGIN +{ + delete (hl2ss::su_result*)reference; +} +HL2SS_ULM_END(void()) + +HL2SS_CLIENT_EXPORT +int32_t su_unpack_item(void* reference, uint32_t index, hl2ss::ulm::su_item& item) +HL2SS_ULM_BEGIN +{ + std::vector& items = ((hl2ss::su_result*)reference)->items; + hl2ss::su_item& m = items[index]; + + item.id = m.id; + item.kind = m.kind; + item.orientation = m.orientation; + item.position = m.position; + item.location = &m.location; + item.alignment = m.alignment; + item.extents = m.extents; + item.meshes = &m.meshes; + item.meshes_count = (uint32_t)m.meshes.size(); + item.collider_meshes = &m.collider_meshes; + item.collider_meshes_count = (uint32_t)m.collider_meshes.size(); + + return 0; +} +HL2SS_ULM_END(-1) + +HL2SS_CLIENT_EXPORT +int32_t su_unpack_item_mesh(void* meshes, uint32_t index, hl2ss::ulm::su_mesh& mesh) +HL2SS_ULM_BEGIN +{ + std::vector& v = *(std::vector*)meshes; + hl2ss::su_mesh& m = v[index]; + + mesh.vertex_positions_data = m.vertex_positions.data(); + mesh.vertex_positions_size = m.vertex_positions.size(); + mesh.triangle_indices_data = m.triangle_indices.data(); + mesh.triangle_indices_size = m.triangle_indices.size(); + + return 0; +} +HL2SS_ULM_END(-1) //------------------------------------------------------------------------------ // Voice Input @@ -540,13 +881,13 @@ HL2SS_ULM_BEGIN HL2SS_ULM_END(-1) HL2SS_CLIENT_EXPORT -int32_t vi_register_commands(void* ipc, bool clear, char const* utf8_array, uint32_t& status) +int32_t vi_register_commands(void* ipc, uint32_t clear, char const* utf8_array, uint32_t& status) HL2SS_ULM_BEGIN { char const* current = utf8_array; std::wstring_convert, char16_t> convert; std::vector commands; - size_t count; + uint64_t count; while ((count = strlen(current)) > 0) { diff --git a/extensions/client_cpp/hl2ss_ulm.h b/extensions/client_cpp/hl2ss_ulm.h index 728e19f17..90f32e071 100644 --- a/extensions/client_cpp/hl2ss_ulm.h +++ b/extensions/client_cpp/hl2ss_ulm.h @@ -312,116 +312,121 @@ struct ray vector_3 direction; }; +struct uint64x2 +{ + uint64_t val[2]; +}; + //------------------------------------------------------------------------------ -// Unpacking +// Packer //------------------------------------------------------------------------------ -struct pv_intrinsics +union v8 { uint8_t b; int8_t c; }; +union v16 { struct { v8 b0, b1; } b; uint16_t w; int16_t s; }; +union v32 { struct { v16 w0, w1; } w; uint32_t d; int32_t i; }; +union v64 { struct { v32 d0, d1; } d; uint64_t q; int64_t l; }; + +constexpr +void push_u8(std::vector& sc, uint8_t byte) { - float fx; - float fy; - float cx; - float cy; -}; + sc.push_back(byte); +} -struct rm_imu_sample +constexpr +void push_u16(std::vector& sc, uint16_t word) { - uint64_t sensor_timestamp; - uint64_t timestamp; - float x; - float y; - float z; - float temperature; -}; + v16 data{word}; -namespace si_valid + push_u8(sc, data.b.b0.b); + push_u8(sc, data.b.b1.b); +} + +constexpr +void push_u32(std::vector& sc, uint32_t dword) { -uint32_t const HEAD = 0x01; -uint32_t const EYE = 0x02; -uint32_t const LEFT = 0x04; -uint32_t const RIGHT = 0x08; + v32 data{dword}; + + push_u16(sc, data.w.w0.w); + push_u16(sc, data.w.w1.w); } -struct si_head_pose +constexpr +void push_u64(std::vector& sc, uint64_t qword) { - vector_3 position; - vector_3 forward; - vector_3 up; -}; + v64 data{qword}; -namespace si_hand_joint_kind + push_u32(sc, data.d.d0.d); + push_u32(sc, data.d.d1.d); +} + +constexpr +void push_float(std::vector& sc, float f) { -uint8_t const Palm = 0; -uint8_t const Wrist = 1; -uint8_t const ThumbMetacarpal = 2; -uint8_t const ThumbProximal = 3; -uint8_t const ThumbDistal = 4; -uint8_t const ThumbTip = 5; -uint8_t const IndexMetacarpal = 6; -uint8_t const IndexProximal = 7; -uint8_t const IndexIntermediate = 8; -uint8_t const IndexDistal = 9; -uint8_t const IndexTip = 10; -uint8_t const MiddleMetacarpal = 11; -uint8_t const MiddleProximal = 12; -uint8_t const MiddleIntermediate = 13; -uint8_t const MiddleDistal = 14; -uint8_t const MiddleTip = 15; -uint8_t const RingMetacarpal = 16; -uint8_t const RingProximal = 17; -uint8_t const RingIntermediate = 18; -uint8_t const RingDistal = 19; -uint8_t const RingTip = 20; -uint8_t const LittleMetacarpal = 21; -uint8_t const LittleProximal = 22; -uint8_t const LittleIntermediate = 23; -uint8_t const LittleDistal = 24; -uint8_t const LittleTip = 25; -uint8_t const TOTAL = 26; + push_u32(sc, *(uint32_t*)&f); } -struct si_hand_joint +constexpr +void push_double(std::vector& sc, double d) { - quaternion orientation; - vector_3 position; - float radius; - int32_t accuracy; -}; + push_u64(sc, *(uint64_t*)&d); +} -struct si_frame +constexpr +void push(std::vector& sc, void const* data, size_t size) { - uint32_t valid; - si_head_pose head_pose; - ray eye_ray; - si_hand_joint left_hand[26]; - si_hand_joint right_hand[26]; -}; + sc.insert(sc.end(), (uint8_t*)data, ((uint8_t*)data) + size); +} -namespace eet_valid +//------------------------------------------------------------------------------ +// Stream Configuration +//------------------------------------------------------------------------------ + +constexpr +uint32_t extended_audio_device_mixer_mode(uint32_t mixer_mode, uint32_t device) { -uint32_t const CALIBRATION = 0x01; -uint32_t const COMBINED_RAY = 0x02; -uint32_t const LEFT_RAY = 0x04; -uint32_t const RIGHT_RAY = 0x08; -uint32_t const LEFT_OPENNESS = 0x10; -uint32_t const RIGHT_OPENNESS = 0x20; -uint32_t const VERGENCE_DISTANCE = 0x40; + uint32_t const DEVICE_BASE = 0x00000004; + return mixer_mode | (DEVICE_BASE * (device + 1)); } -struct eet_frame +//------------------------------------------------------------------------------ +// Decoders +//------------------------------------------------------------------------------ + +struct rm_vlc_metadata { + uint64_t sensor_ticks; + uint64_t exposure; + uint32_t gain; + uint32_t _reserved; +}; + +struct rm_depth_ahat_metadata +{ + uint64_t sensor_ticks; +}; + +struct rm_depth_longthrow_metadata +{ + uint64_t sensor_ticks; +}; + +struct pv_metadata +{ + vector_2 f; + vector_2 c; + uint64_t exposure_time; + uint64x2 exposure_compensation; + uint32_t lens_position; + uint32_t focus_state; + uint32_t iso_speed; + uint32_t white_balance; + vector_2 iso_gains; + vector_3 white_balance_gains; uint32_t _reserved; - ray combined_ray; - ray left_ray; - ray right_ray; - float left_openness; - float right_openness; - float vergence_distance; - uint32_t valid; }; //------------------------------------------------------------------------------ -// Calibration +// Mode 2 Data Acquisition //------------------------------------------------------------------------------ struct calibration_rm_vlc @@ -464,8 +469,44 @@ struct calibration_pv float tangential_distortion[2]; float projection[4][4]; float extrinsics[4][4]; + float intrinsics_mf[4]; + float extrinsics_mf[7]; }; +//------------------------------------------------------------------------------ +// Port Information +//------------------------------------------------------------------------------ + +constexpr +char const* get_port_name(uint16_t port) +{ + switch (port) + { + case hl2ss::stream_port::RM_VLC_LEFTFRONT: return "rm_vlc_leftfront"; + case hl2ss::stream_port::RM_VLC_LEFTLEFT: return "rm_vlc_leftleft"; + case hl2ss::stream_port::RM_VLC_RIGHTFRONT: return "rm_vlc_rightfront"; + case hl2ss::stream_port::RM_VLC_RIGHTRIGHT: return "rm_vlc_rightright"; + case hl2ss::stream_port::RM_DEPTH_AHAT: return "rm_depth_ahat"; + case hl2ss::stream_port::RM_DEPTH_LONGTHROW: return "rm_depth_longthrow"; + case hl2ss::stream_port::RM_IMU_ACCELEROMETER: return "rm_imu_accelerometer"; + case hl2ss::stream_port::RM_IMU_GYROSCOPE: return "rm_imu_gyroscope"; + case hl2ss::stream_port::RM_IMU_MAGNETOMETER: return "rm_imu_magnetometer"; + case hl2ss::ipc_port::REMOTE_CONFIGURATION: return "remote_configuration"; + case hl2ss::stream_port::PERSONAL_VIDEO: return "personal_video"; + case hl2ss::stream_port::MICROPHONE: return "microphone"; + case hl2ss::stream_port::SPATIAL_INPUT: return "spatial_input"; + case hl2ss::ipc_port::SPATIAL_MAPPING: return "spatial_mapping"; + case hl2ss::ipc_port::SCENE_UNDERSTANDING: return "scene_understanding"; + case hl2ss::ipc_port::VOICE_INPUT: return "voice_input"; + case hl2ss::ipc_port::UNITY_MESSAGE_QUEUE: return "unity_message_queue"; + case hl2ss::stream_port::EXTENDED_EYE_TRACKER: return "extended_eye_tracker"; + case hl2ss::stream_port::EXTENDED_AUDIO: return "extended_audio"; + case hl2ss::stream_port::EXTENDED_VIDEO: return "extended_video"; + case hl2ss::ipc_port::GUEST_MESSAGE_QUEUE: return "guest_message_queue"; + default: return nullptr; + } +} + //------------------------------------------------------------------------------ // Remote Configuration //------------------------------------------------------------------------------ @@ -563,6 +604,12 @@ uint32_t const Min = 100; uint32_t const Max = 3200; } +namespace pv_backlight_compensation_state +{ +uint32_t const Disable = 0; +uint32_t const Enable = 1; +} + namespace pv_capture_scene_mode { uint32_t const Auto = 0; @@ -579,10 +626,50 @@ uint32_t const NightPortrait = 11; uint32_t const Backlit = 12; } -namespace pv_backlight_compensation_state +namespace pv_media_capture_optimization { -uint32_t const Disable = 0; -uint32_t const Enable = 1; +uint32_t const Default = 0; +uint32_t const Quality = 1; +uint32_t const Latency = 2; +uint32_t const Power = 3; +uint32_t const LatencyThenQuality = 4; +uint32_t const LatencyThenPower = 5; +uint32_t const PowerAndQuality = 6; +} + +namespace pv_capture_use +{ +uint32_t const NotSet = 0; +uint32_t const Photo = 1; +uint32_t const Video = 2; +} + +namespace pv_optical_image_stabilization_mode +{ +uint32_t const Off = 0; +uint32_t const On = 1; +} + +namespace pv_hdr_video_mode +{ +uint32_t const Off = 0; +uint32_t const On = 1; +uint32_t const Auto = 2; +} + +namespace pv_region_of_interest_type +{ +uint32_t const Unknown = 0; +uint32_t const Face = 1; +} + +namespace interface_priority +{ +int32_t const LOWEST = -2; +int32_t const BELOW_NORMAL = -1; +int32_t const NORMAL = 0; +int32_t const ABOVE_NORMAL = 1; +int32_t const HIGHEST = 2; } struct version @@ -594,13 +681,222 @@ struct version // Spatial Mapping //------------------------------------------------------------------------------ -// TODO +namespace sm_vertex_position_format +{ +uint32_t const R32G32B32A32Float = 2; +uint32_t const R16G16B16A16IntNormalized = 13; +} + +namespace sm_triangle_index_format +{ +uint32_t const R16UInt = 57; +uint32_t const R32Uint = 42; +} + +namespace sm_vertex_normal_format +{ +uint32_t const R32G32B32A32Float = 2; +uint32_t const R8G8B8A8IntNormalized = 31; +} + +namespace sm_volume_type +{ +uint32_t const Box = 0; +uint32_t const Frustum = 1; +uint32_t const OrientedBox = 2; +uint32_t const Sphere = 3; +} + +struct sm_box +{ + vector_3 center; + vector_3 extents; +}; + +struct sm_frustum +{ + plane p_near; + plane p_far; + plane p_right; + plane p_left; + plane p_top; + plane p_bottom; +}; + +struct sm_oriented_box +{ + vector_3 center; + vector_3 extents; + quaternion orientation; +}; + +struct sm_sphere +{ + vector_3 center; + float radius; +}; + +struct guid +{ + uint64_t l; + uint64_t h; +}; + +struct sm_surface_info +{ + guid id; + uint64_t update_time; +}; + +class sm_bounding_volume +{ +private: + std::vector m_data; + uint32_t m_count; + +public: + sm_bounding_volume() + { + m_count = 0; + } + + sm_bounding_volume(uint32_t count, uint8_t const* data, size_t size) + { + m_count = count; + m_data = { data, data + size }; + } + + void add_box(sm_box box) + { + m_count++; + push_u32(m_data, sm_volume_type::Box); + push(m_data, &box, sizeof(box)); + } + + void add_frustum(sm_frustum frustum) + { + m_count++; + push_u32(m_data, sm_volume_type::Frustum); + push(m_data, &frustum, sizeof(frustum)); + } + + void add_oriented_box(sm_oriented_box oriented_box) + { + m_count++; + push_u32(m_data, sm_volume_type::OrientedBox); + push(m_data, &oriented_box, sizeof(oriented_box)); + } + + void add_sphere(sm_sphere sphere) + { + m_count++; + push_u32(m_data, sm_volume_type::Sphere); + push(m_data, &sphere, sizeof(sphere)); + } + + uint32_t get_count() const + { + return m_count; + } + + uint8_t const* get_data() const + { + return m_data.data(); + } + + size_t get_size() const + { + return m_data.size(); + } +}; + +class sm_mesh_task +{ +private: + std::vector m_data; + uint32_t m_count; + +public: + sm_mesh_task() + { + m_count = 0; + } + + sm_mesh_task(uint32_t count, uint8_t const* data, size_t size) + { + m_count = count; + m_data = { data, data + size }; + } + + void add_task(guid id, double max_triangles_per_cubic_meter, uint32_t vertex_position_format, uint32_t triangle_index_format, uint32_t vertex_normal_format, bool include_vertex_normals, bool include_bounds) + { + m_count++; + push_u64(m_data, id.l); + push_u64(m_data, id.h); + push_double(m_data, max_triangles_per_cubic_meter); + push_u32(m_data, vertex_position_format); + push_u32(m_data, triangle_index_format); + push_u32(m_data, vertex_normal_format); + push_u32(m_data, (1*include_vertex_normals) | (2*include_bounds)); + } + + uint32_t get_count() const + { + return m_count; + } + + uint8_t const* get_data() const + { + return m_data.data(); + } + + size_t get_size() const + { + return m_data.size(); + } +}; //------------------------------------------------------------------------------ // Scene Understanding //------------------------------------------------------------------------------ -// TODO +namespace su_mesh_lod +{ +uint32_t const Coarse = 0; +uint32_t const Medium = 1; +uint32_t const Fine = 2; +uint32_t const Unlimited = 255; +} + +namespace su_kind_flag +{ +uint8_t const Background = 1; +uint8_t const Wall = 2; +uint8_t const Floor = 4; +uint8_t const Ceiling = 8; +uint8_t const Platform = 16; +uint8_t const Unknown = 32; +uint8_t const World = 64; +uint8_t const CompletelyInferred = 128; +} + +namespace su_create +{ +uint8_t const New = 0; +uint8_t const NewFromPrevious = 1; +} + +namespace su_kind +{ +int32_t const Background = 0; +int32_t const Wall = 1; +int32_t const Floor = 2; +int32_t const Ceiling = 3; +int32_t const Platform = 4; +int32_t const Unknown = 247; +int32_t const World = 248; +int32_t const CompletelyInferred = 249; +} //------------------------------------------------------------------------------ // Voice Input @@ -624,68 +920,181 @@ struct vi_result }; //------------------------------------------------------------------------------ -// Stream Configuration +// Unity Message Queue //------------------------------------------------------------------------------ -constexpr -char const* get_port_name(uint16_t port) +class umq_command_buffer { - switch (port) +private: + std::vector m_buffer; + uint32_t m_count; + +public: + umq_command_buffer() { - case hl2ss::stream_port::RM_VLC_LEFTFRONT: return "rm_vlc_leftfront"; - case hl2ss::stream_port::RM_VLC_LEFTLEFT: return "rm_vlc_leftleft"; - case hl2ss::stream_port::RM_VLC_RIGHTFRONT: return "rm_vlc_rightfront"; - case hl2ss::stream_port::RM_VLC_RIGHTRIGHT: return "rm_vlc_rightright"; - case hl2ss::stream_port::RM_DEPTH_AHAT: return "rm_depth_ahat"; - case hl2ss::stream_port::RM_DEPTH_LONGTHROW: return "rm_depth_longthrow"; - case hl2ss::stream_port::RM_IMU_ACCELEROMETER: return "rm_imu_accelerometer"; - case hl2ss::stream_port::RM_IMU_GYROSCOPE: return "rm_imu_gyroscope"; - case hl2ss::stream_port::RM_IMU_MAGNETOMETER: return "rm_imu_magnetometer"; - case hl2ss::ipc_port::REMOTE_CONFIGURATION: return "remote_configuration"; - case hl2ss::stream_port::PERSONAL_VIDEO: return "personal_video"; - case hl2ss::stream_port::MICROPHONE: return "microphone"; - case hl2ss::stream_port::SPATIAL_INPUT: return "spatial_input"; - case hl2ss::ipc_port::SPATIAL_MAPPING: return "spatial_mapping"; - case hl2ss::ipc_port::SCENE_UNDERSTANDING: return "scene_understanding"; - case hl2ss::ipc_port::VOICE_INPUT: return "voice_input"; - case hl2ss::ipc_port::UNITY_MESSAGE_QUEUE: return "unity_message_queue"; - case hl2ss::stream_port::EXTENDED_EYE_TRACKER: return "extended_eye_tracker"; - case hl2ss::stream_port::EXTENDED_AUDIO: return "extended_audio"; - case hl2ss::stream_port::EXTENDED_VIDEO: return "extended_video"; - case hl2ss::ipc_port::GUEST_MESSAGE_QUEUE: return "guest_message_queue"; - default: return nullptr; + m_count = 0; } -} -constexpr -uint32_t extended_audio_device_mixer_mode(uint32_t mixer_mode, uint32_t device) -{ - uint32_t const DEVICE_BASE = 0x00000004; - return mixer_mode | (DEVICE_BASE * (device + 1)); -} + void add(uint32_t id, void const* data, size_t size) + { + push_u32(m_buffer, id); + push_u32(m_buffer, (uint32_t)size); + push(m_buffer, data, size); + m_count++; + } + + void clear() + { + m_buffer.clear(); + m_count = 0; + } + + uint8_t const* data() + { + return m_buffer.data(); + } + + size_t size() + { + return m_buffer.size(); + } + + uint32_t count() + { + return m_count; + } +}; //------------------------------------------------------------------------------ // Unpacking //------------------------------------------------------------------------------ +struct rm_imu_sample +{ + uint64_t sensor_timestamp; + uint64_t timestamp; + float x; + float y; + float z; + float temperature; +}; + +namespace pv_focus_state +{ +uint32_t const UNINITIALIZED = 0; +uint32_t const LOST = 1; +uint32_t const SEARCHING = 2; +uint32_t const FOCUSED = 3; +uint32_t const FAILED = 4; +} + +namespace si_valid +{ +uint32_t const HEAD = 0x01; +uint32_t const EYE = 0x02; +uint32_t const LEFT = 0x04; +uint32_t const RIGHT = 0x08; +} + +struct si_head_pose +{ + vector_3 position; + vector_3 forward; + vector_3 up; +}; + +namespace si_hand_joint_kind +{ +uint8_t const Palm = 0; +uint8_t const Wrist = 1; +uint8_t const ThumbMetacarpal = 2; +uint8_t const ThumbProximal = 3; +uint8_t const ThumbDistal = 4; +uint8_t const ThumbTip = 5; +uint8_t const IndexMetacarpal = 6; +uint8_t const IndexProximal = 7; +uint8_t const IndexIntermediate = 8; +uint8_t const IndexDistal = 9; +uint8_t const IndexTip = 10; +uint8_t const MiddleMetacarpal = 11; +uint8_t const MiddleProximal = 12; +uint8_t const MiddleIntermediate = 13; +uint8_t const MiddleDistal = 14; +uint8_t const MiddleTip = 15; +uint8_t const RingMetacarpal = 16; +uint8_t const RingProximal = 17; +uint8_t const RingIntermediate = 18; +uint8_t const RingDistal = 19; +uint8_t const RingTip = 20; +uint8_t const LittleMetacarpal = 21; +uint8_t const LittleProximal = 22; +uint8_t const LittleIntermediate = 23; +uint8_t const LittleDistal = 24; +uint8_t const LittleTip = 25; +uint8_t const TOTAL = 26; +} + +struct si_hand_joint +{ + quaternion orientation; + vector_3 position; + float radius; + int32_t accuracy; +}; + +struct si_frame +{ + uint32_t valid; + si_head_pose head_pose; + ray eye_ray; + si_hand_joint left_hand[26]; + si_hand_joint right_hand[26]; +}; + +namespace eet_valid +{ +uint32_t const CALIBRATION = 0x01; +uint32_t const COMBINED_RAY = 0x02; +uint32_t const LEFT_RAY = 0x04; +uint32_t const RIGHT_RAY = 0x08; +uint32_t const LEFT_OPENNESS = 0x10; +uint32_t const RIGHT_OPENNESS = 0x20; +uint32_t const VERGENCE_DISTANCE = 0x40; +} + +struct eet_frame +{ + uint32_t _reserved; + ray combined_ray; + ray left_ray; + ray right_ray; + float left_openness; + float right_openness; + float vergence_distance; + uint32_t valid; +}; + constexpr -void unpack_rm_vlc(uint8_t* payload, uint8_t*& image) +void unpack_rm_vlc(uint8_t* payload, uint8_t*& image, rm_vlc_metadata*& metadata) { - image = payload; + image = payload; + metadata = (rm_vlc_metadata*)(payload + parameters_rm_vlc::PIXELS); } constexpr -void unpack_rm_depth_ahat(uint8_t* payload, uint16_t*& depth, uint16_t*& ab) +void unpack_rm_depth_ahat(uint8_t* payload, uint16_t*& depth, uint16_t*& ab, rm_depth_ahat_metadata*& metadata) { - depth = (uint16_t*)(payload); - ab = (uint16_t*)(payload + (parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t))); + depth = (uint16_t*) (payload); + ab = (uint16_t*) (payload + ( parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t))); + metadata = (rm_depth_ahat_metadata*)(payload + (2 * parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t))); } constexpr -void unpack_rm_depth_longthrow(uint8_t* payload, uint16_t*& depth, uint16_t*& ab) +void unpack_rm_depth_longthrow(uint8_t* payload, uint16_t*& depth, uint16_t*& ab, rm_depth_longthrow_metadata*& metadata) { - depth = (uint16_t*)(payload); - ab = (uint16_t*)(payload + (parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t))); + depth = (uint16_t*) (payload); + ab = (uint16_t*) (payload + ( parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t))); + metadata = (rm_depth_longthrow_metadata*)(payload + (2 * parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t))); } constexpr @@ -695,10 +1104,10 @@ void unpack_rm_imu(uint8_t* payload, rm_imu_sample*& samples) } constexpr -void unpack_pv(uint8_t* payload, uint64_t size, uint8_t*& image, pv_intrinsics*& intrinsics) +void unpack_pv(uint8_t* payload, size_t size, uint8_t*& image, pv_metadata*& metadata) { - image = payload; - intrinsics = (pv_intrinsics*)(payload + size - sizeof(pv_intrinsics)); + image = payload; + metadata = (pv_metadata*)(payload + size - sizeof(pv_metadata)); } constexpr @@ -771,13 +1180,82 @@ namespace ulm struct packet { + int64_t frame_stamp; uint64_t timestamp; uint32_t sz_payload; - uint32_t _reserved; + int32_t status; uint8_t* payload; matrix_4x4* pose; }; +struct sm_mesh +{ + uint32_t status; + vector_3 vertex_position_scale; + uint64_t bounds_size; + uint64_t vertex_positions_size; + uint64_t triangle_indices_size; + uint64_t vertex_normals_size; + matrix_4x4* pose; + uint8_t* bounds_data; + uint8_t* vertex_positions_data; + uint8_t* triangle_indices_data; + uint8_t* vertex_normals_data; +}; + +struct su_mesh +{ + uint64_t vertex_positions_size; + uint64_t triangle_indices_size; + uint8_t* vertex_positions_data; + uint8_t* triangle_indices_data; +}; + +struct su_item +{ + guid id; + int32_t kind; + uint32_t _reserved; + quaternion orientation; + vector_3 position; + int32_t alignment; + vector_2 extents; + uint32_t meshes_count; + uint32_t collider_meshes_count; + matrix_4x4* location; + void* meshes; + void* collider_meshes; +}; + +struct su_result_header +{ + uint32_t status; + uint32_t count; + matrix_4x4* extrinsics; + matrix_4x4* pose; +}; + +struct su_task +{ + bool enable_quads; + bool enable_meshes; + bool enable_only_observed; + bool enable_world_mesh; + uint32_t mesh_lod; + float query_radius; + uint8_t create_mode; + uint8_t kind_flags; + bool get_orientation; + bool get_position; + bool get_location_matrix; + bool get_quad; + bool get_meshes; + bool get_collider_meshes; + uint32_t _reserved; + uint64_t guid_list_size; + guid* guid_list_data; +}; + struct gmq_message { uint32_t command; @@ -856,13 +1334,13 @@ void close_ipc(void* ipc); //----------------------------------------------------------------------------- HL2SS_CLIENT_IMPORT -int32_t get_by_index(void* source, int64_t& frame_stamp, int32_t& status, void*& frame, hl2ss::ulm::packet& packet); +void* get_by_index(void* source, int64_t frame_stamp, hl2ss::ulm::packet& packet); HL2SS_CLIENT_IMPORT -int32_t get_by_timestamp(void* source, uint64_t timestamp, int32_t time_preference, int32_t tiebreak_right, int64_t& frame_stamp, int32_t& status, void*& frame, hl2ss::ulm::packet& packet); +void* get_by_timestamp(void* source, uint64_t timestamp, int32_t time_preference, int32_t tiebreak_right, hl2ss::ulm::packet& packet); HL2SS_CLIENT_IMPORT -void release_frame(void* frame); +void release_packet(void* reference); //----------------------------------------------------------------------------- // Control @@ -879,19 +1357,34 @@ int32_t stop_subsystem_pv(char const* host, uint16_t port); //----------------------------------------------------------------------------- HL2SS_CLIENT_IMPORT -int32_t download_calibration_rm_vlc(char const* host, uint16_t port, calibration_rm_vlc& calibration); +void* download_calibration_rm_vlc(char const* host, uint16_t port, hl2ss::calibration_rm_vlc*& calibration); + +HL2SS_CLIENT_IMPORT +void* download_calibration_rm_depth_ahat(char const* host, uint16_t port, hl2ss::calibration_rm_depth_ahat*& calibration); + +HL2SS_CLIENT_IMPORT +void* download_calibration_rm_depth_longthrow(char const* host, uint16_t port, calibration_rm_depth_longthrow*& calibration); + +HL2SS_CLIENT_IMPORT +void* download_calibration_rm_imu(char const* host, uint16_t port, calibration_rm_imu*& calibration); HL2SS_CLIENT_IMPORT -int32_t download_calibration_rm_depth_ahat(char const* host, uint16_t port, calibration_rm_depth_ahat& calibration); +void* download_calibration_pv(char const* host, uint16_t port, uint16_t width, uint16_t height, uint8_t framerate, calibration_pv*& calibration); HL2SS_CLIENT_IMPORT -int32_t download_calibration_rm_depth_longthrow(char const* host, uint16_t port, calibration_rm_depth_longthrow& calibration); +void release_calibration_rm_vlc(void* reference); HL2SS_CLIENT_IMPORT -int32_t download_calibration_rm_imu(char const* host, uint16_t port, calibration_rm_imu& calibration); +void release_calibration_rm_depth_ahat(void* reference); HL2SS_CLIENT_IMPORT -int32_t download_calibration_pv(char const* host, uint16_t port, uint16_t width, uint16_t height, uint8_t framerate, calibration_pv& calibration); +void release_calibration_rm_depth_longthrow(void* reference); + +HL2SS_CLIENT_IMPORT +void release_calibration_rm_imu(void* reference); + +HL2SS_CLIENT_IMPORT +void release_calibration_pv(void* reference); //------------------------------------------------------------------------------ // Remote Configuration @@ -942,17 +1435,67 @@ int32_t rc_set_pv_scene_mode(void* ipc, uint32_t mode); HL2SS_CLIENT_IMPORT int32_t rc_set_flat_mode(void* ipc, uint32_t mode); +HL2SS_CLIENT_IMPORT +int32_t rc_set_rm_eye_selection(void* ipc, uint32_t enable); + +HL2SS_CLIENT_IMPORT +int32_t rc_set_pv_desired_optimization(void* ipc, uint32_t mode); + +HL2SS_CLIENT_IMPORT +int32_t rc_set_pv_primary_use(void* ipc, uint32_t mode); + +HL2SS_CLIENT_IMPORT +int32_t rc_set_pv_optical_image_stabilization(void* ipc, uint32_t mode); + +HL2SS_CLIENT_IMPORT +int32_t rc_set_pv_hdr_video(void* ipc, uint32_t mode); + +HL2SS_CLIENT_IMPORT +int32_t rc_set_pv_regions_of_interest(void* ipc, uint32_t clear, uint32_t set, uint32_t auto_exposure, uint32_t auto_focus, uint32_t bounds_normalized, uint32_t type, uint32_t weight, float x, float y, float w, float h); + +HL2SS_CLIENT_IMPORT +int32_t rc_set_interface_priority(void* ipc, uint16_t port, int32_t priority); + //------------------------------------------------------------------------------ // Spatial Mapping //------------------------------------------------------------------------------ -// TODO: +HL2SS_CLIENT_IMPORT +int32_t sm_create_observer(void* ipc); + +HL2SS_CLIENT_IMPORT +int32_t sm_set_volumes(void* ipc, uint32_t count, uint8_t const* data, uint64_t size); + +HL2SS_CLIENT_IMPORT +void* sm_get_observed_surfaces(void* ipc, uint64_t& size, hl2ss::sm_surface_info*& data); + +HL2SS_CLIENT_IMPORT +void sm_release_surfaces(void* reference); + +HL2SS_CLIENT_IMPORT +void* sm_get_meshes(void* ipc, uint32_t count, uint8_t const* data, uint64_t size, uint32_t threads); + +HL2SS_CLIENT_IMPORT +void sm_release_meshes(void* reference); + +HL2SS_CLIENT_IMPORT +int32_t sm_unpack_mesh(void* reference, uint32_t index, hl2ss::ulm::sm_mesh& mesh); //------------------------------------------------------------------------------ // Scene Understanding //------------------------------------------------------------------------------ -// TODO: +HL2SS_CLIENT_IMPORT +void* su_query(void* ipc, hl2ss::ulm::su_task const& task, hl2ss::ulm::su_result_header& header); + +HL2SS_CLIENT_IMPORT +void su_release(void* reference); + +HL2SS_CLIENT_IMPORT +int32_t su_unpack_item(void* reference, uint32_t index, hl2ss::ulm::su_item& item); + +HL2SS_CLIENT_IMPORT +int32_t su_unpack_item_mesh(void* meshes, uint32_t index, hl2ss::ulm::su_mesh& mesh); //------------------------------------------------------------------------------ // Voice Input @@ -962,7 +1505,7 @@ HL2SS_CLIENT_IMPORT int32_t vi_create_recognizer(void* ipc); HL2SS_CLIENT_IMPORT -int32_t vi_register_commands(void* ipc, bool clear, char const* utf8_array, uint32_t& status); +int32_t vi_register_commands(void* ipc, uint32_t clear, char const* utf8_array, uint32_t& status); HL2SS_CLIENT_IMPORT int32_t vi_start(void* ipc); diff --git a/extensions/client_matlab/+hl2ss/+ms/sink_pv.m b/extensions/client_matlab/+hl2ss/+ms/sink_pv.m index bc8d0848f..373d7d798 100644 --- a/extensions/client_matlab/+hl2ss/+ms/sink_pv.m +++ b/extensions/client_matlab/+hl2ss/+ms/sink_pv.m @@ -70,6 +70,9 @@ function setupImpl(obj) 'timestamp', zeros([1, 1], 'uint64'), ... 'image', zeros(obj.getImageSize(), 'uint8' ), ... 'intrinsics', zeros([4, 1], 'single'), ... + 'exposure', zeros([3, 1], 'uint64'), ... + 'imaging', zeros([4, 1], 'uint32'), ... + 'gains', zeros([5, 1], 'single'), ... 'pose', zeros([4, 4], 'single')); coder.extrinsic('hl2ss_matlab') @@ -80,7 +83,7 @@ function setupImpl(obj) obj.client.open() end - function [frame_index, status, timestamp, image, intrinsics, pose] = stepImpl(obj, sync, index) + function [frame_index, status, timestamp, image, intrinsics, pose, exposure, imaging, gains] = stepImpl(obj, sync, index) if (sync <= 0) response = obj.client.get_packet_by_index(index); else @@ -97,6 +100,9 @@ function setupImpl(obj) timestamp = data.timestamp; image = data.image; intrinsics = data.intrinsics; + exposure = data.exposure; + imaging = data.imaging; + gains = data.gains; pose = data.pose; end @@ -109,40 +115,52 @@ function releaseImpl(obj) obj.client.stop_subsystem() end - function [out1, out2, out3, out4, out5, out6] = getOutputSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8, out9] = getOutputSizeImpl(obj) out1 = [1, 1]; out2 = [1, 1]; out3 = [1, 1]; out4 = obj.getImageSize(); out5 = [4, 1]; out6 = [4, 4]; + out7 = [3, 1]; + out8 = [4, 1]; + out9 = [5, 1]; end - function [out1, out2, out3, out4, out5, out6] = getOutputDataTypeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8, out9] = getOutputDataTypeImpl(obj) out1 = 'int64'; out2 = 'int32'; out3 = 'uint64'; out4 = 'uint8'; out5 = 'single'; out6 = 'single'; + out7 = 'uint64'; + out8 = 'uint32'; + out9 = 'single'; end - function [out1, out2, out3, out4, out5, out6] = isOutputComplexImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8, out9] = isOutputComplexImpl(obj) out1 = false; out2 = false; out3 = false; out4 = false; out5 = false; out6 = false; + out7 = false; + out8 = false; + out9 = false; end - function [out1, out2, out3, out4, out5, out6] = isOutputFixedSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8, out9] = isOutputFixedSizeImpl(obj) out1 = true; out2 = true; out3 = true; out4 = true; out5 = true; out6 = true; + out7 = true; + out8 = true; + out9 = true; end function sts = getSampleTimeImpl(obj) diff --git a/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_ahat.m b/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_ahat.m index a79f99a63..01399ad29 100644 --- a/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_ahat.m +++ b/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_ahat.m @@ -39,12 +39,13 @@ function setupImpl(obj) obj.definition_rm_depth_ahat = ... - struct('frame_index', zeros([1, 1], 'int64' ), ... - 'status', zeros([1, 1], 'int32' ), ... - 'timestamp', zeros([1, 1], 'uint64'), ... - 'depth', zeros(obj.getImageSize(), 'uint16'), ... - 'ab', zeros(obj.getImageSize(), 'uint16'), ... - 'pose', zeros([4, 4], 'single')); + struct('frame_index', zeros([1, 1], 'int64' ), ... + 'status', zeros([1, 1], 'int32' ), ... + 'timestamp', zeros([1, 1], 'uint64'), ... + 'depth', zeros(obj.getImageSize(), 'uint16'), ... + 'ab', zeros(obj.getImageSize(), 'uint16'), ... + 'sensor_ticks', zeros([1, 1], 'uint64'), ... + 'pose', zeros([4, 4], 'single')); coder.extrinsic('hl2ss_matlab') @@ -53,7 +54,7 @@ function setupImpl(obj) obj.client.open() end - function [frame_index, status, timestamp, depth, ab, pose] = stepImpl(obj, sync, index) + function [frame_index, status, timestamp, depth, ab, pose, sensor_ticks] = stepImpl(obj, sync, index) if (sync <= 0) response = obj.client.get_packet_by_index(index); else @@ -65,12 +66,13 @@ function setupImpl(obj) data = obj.definition_rm_depth_ahat; data = hl2ss.ms.unpack_rm_depth_ahat(response, obj.mode); - frame_index = data.frame_index; - status = data.status; - timestamp = data.timestamp; - depth = data.depth; - ab = data.ab; - pose = data.pose; + frame_index = data.frame_index; + status = data.status; + timestamp = data.timestamp; + depth = data.depth; + ab = data.ab; + sensor_ticks = data.sensor_ticks; + pose = data.pose; end function resetImpl(obj) @@ -81,40 +83,44 @@ function releaseImpl(obj) obj.client.close(); end - function [out1, out2, out3, out4, out5, out6] = getOutputSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = getOutputSizeImpl(obj) out1 = [1, 1]; out2 = [1, 1]; out3 = [1, 1]; out4 = obj.getImageSize(); out5 = obj.getImageSize(); out6 = [4, 4]; + out7 = [1, 1]; end - function [out1, out2, out3, out4, out5, out6] = getOutputDataTypeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = getOutputDataTypeImpl(obj) out1 = 'int64'; out2 = 'int32'; out3 = 'uint64'; out4 = 'uint16'; out5 = 'uint16'; out6 = 'single'; + out7 = 'uint64'; end - function [out1, out2, out3, out4, out5, out6] = isOutputComplexImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = isOutputComplexImpl(obj) out1 = false; out2 = false; out3 = false; out4 = false; out5 = false; out6 = false; + out7 = false; end - function [out1, out2, out3, out4, out5, out6] = isOutputFixedSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = isOutputFixedSizeImpl(obj) out1 = true; out2 = true; out3 = true; out4 = true; out5 = true; out6 = true; + out7 = true; end function sts = getSampleTimeImpl(obj) diff --git a/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_longthrow.m b/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_longthrow.m index ba40c69de..90f55cbff 100644 --- a/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_longthrow.m +++ b/extensions/client_matlab/+hl2ss/+ms/sink_rm_depth_longthrow.m @@ -35,12 +35,13 @@ function setupImpl(obj) obj.definition_rm_depth_longthrow = ... - struct('frame_index', zeros([1, 1], 'int64' ), ... - 'status', zeros([1, 1], 'int32' ), ... - 'timestamp', zeros([1, 1], 'uint64'), ... - 'depth', zeros(obj.getImageSize(), 'uint16'), ... - 'ab', zeros(obj.getImageSize(), 'uint16'), ... - 'pose', zeros([4, 4], 'single')); + struct('frame_index', zeros([1, 1], 'int64' ), ... + 'status', zeros([1, 1], 'int32' ), ... + 'timestamp', zeros([1, 1], 'uint64'), ... + 'depth', zeros(obj.getImageSize(), 'uint16'), ... + 'ab', zeros(obj.getImageSize(), 'uint16'), ... + 'sensor_ticks', zeros([1, 1], 'uint64'), ... + 'pose', zeros([4, 4], 'single')); coder.extrinsic('hl2ss_matlab') @@ -49,7 +50,7 @@ function setupImpl(obj) obj.client.open() end - function [frame_index, status, timestamp, depth, ab, pose] = stepImpl(obj, sync, index) + function [frame_index, status, timestamp, depth, ab, pose, sensor_ticks] = stepImpl(obj, sync, index) if (sync <= 0) response = obj.client.get_packet_by_index(index); else @@ -61,12 +62,13 @@ function setupImpl(obj) data = obj.definition_rm_depth_longthrow; data = hl2ss.ms.unpack_rm_depth_longthrow(response, obj.mode); - frame_index = data.frame_index; - status = data.status; - timestamp = data.timestamp; - depth = data.depth; - ab = data.ab; - pose = data.pose; + frame_index = data.frame_index; + status = data.status; + timestamp = data.timestamp; + depth = data.depth; + ab = data.ab; + sensor_ticks = data.sensor_ticks; + pose = data.pose; end function resetImpl(obj) @@ -77,40 +79,44 @@ function releaseImpl(obj) obj.client.close(); end - function [out1, out2, out3, out4, out5, out6] = getOutputSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = getOutputSizeImpl(obj) out1 = [1, 1]; out2 = [1, 1]; out3 = [1, 1]; out4 = obj.getImageSize(); out5 = obj.getImageSize(); out6 = [4, 4]; + out7 = [1, 1]; end - function [out1, out2, out3, out4, out5, out6] = getOutputDataTypeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = getOutputDataTypeImpl(obj) out1 = 'int64'; out2 = 'int32'; out3 = 'uint64'; out4 = 'uint16'; out5 = 'uint16'; out6 = 'single'; + out7 = 'uint64'; end - function [out1, out2, out3, out4, out5, out6] = isOutputComplexImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = isOutputComplexImpl(obj) out1 = false; out2 = false; out3 = false; out4 = false; out5 = false; out6 = false; + out7 = false; end - function [out1, out2, out3, out4, out5, out6] = isOutputFixedSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7] = isOutputFixedSizeImpl(obj) out1 = true; out2 = true; out3 = true; out4 = true; out5 = true; out6 = true; + out7 = true; end function sts = getSampleTimeImpl(obj) diff --git a/extensions/client_matlab/+hl2ss/+ms/sink_rm_vlc.m b/extensions/client_matlab/+hl2ss/+ms/sink_rm_vlc.m index 17d383071..1a7ce968d 100644 --- a/extensions/client_matlab/+hl2ss/+ms/sink_rm_vlc.m +++ b/extensions/client_matlab/+hl2ss/+ms/sink_rm_vlc.m @@ -38,11 +38,14 @@ function setupImpl(obj) obj.definition_rm_vlc = ... - struct('frame_index', zeros([1, 1], 'int64' ), ... - 'status', zeros([1, 1], 'int32' ), ... - 'timestamp', zeros([1, 1], 'uint64'), ... - 'image', zeros(obj.getImageSize(), 'uint8' ), ... - 'pose', zeros([4, 4], 'single')); + struct('frame_index', zeros([1, 1], 'int64' ), ... + 'status', zeros([1, 1], 'int32' ), ... + 'timestamp', zeros([1, 1], 'uint64'), ... + 'image', zeros(obj.getImageSize(), 'uint8' ), ... + 'sensor_ticks', zeros([1, 1], 'uint64'), ... + 'exposure', zeros([1, 1], 'uint64'), ... + 'gain', zeros([1, 1], 'uint32'), ... + 'pose', zeros([4, 4], 'single')); coder.extrinsic('hl2ss_matlab') @@ -51,7 +54,7 @@ function setupImpl(obj) obj.client.open() end - function [frame_index, status, timestamp, image, pose] = stepImpl(obj, sync, index) + function [frame_index, status, timestamp, image, pose, sensor_ticks, exposure, gain] = stepImpl(obj, sync, index) if (sync <= 0) response = obj.client.get_packet_by_index(index); else @@ -63,11 +66,14 @@ function setupImpl(obj) data = obj.definition_rm_vlc; data = hl2ss.ms.unpack_rm_vlc(response, obj.mode); - frame_index = data.frame_index; - status = data.status; - timestamp = data.timestamp; - image = data.image; - pose = data.pose; + frame_index = data.frame_index; + status = data.status; + timestamp = data.timestamp; + image = data.image; + sensor_ticks = data.sensor_ticks; + exposure = data.exposure; + gain = data.gain; + pose = data.pose; end function resetImpl(obj) @@ -78,36 +84,48 @@ function releaseImpl(obj) obj.client.close(); end - function [out1, out2, out3, out4, out5] = getOutputSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8] = getOutputSizeImpl(obj) out1 = [1, 1]; out2 = [1, 1]; out3 = [1, 1]; out4 = obj.getImageSize(); out5 = [4, 4]; + out6 = [1, 1]; + out7 = [1, 1]; + out8 = [1, 1]; end - function [out1, out2, out3, out4, out5] = getOutputDataTypeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8] = getOutputDataTypeImpl(obj) out1 = 'int64'; out2 = 'int32'; out3 = 'uint64'; out4 = 'uint8'; out5 = 'single'; + out6 = 'uint64'; + out7 = 'uint64'; + out8 = 'uint32'; end - function [out1, out2, out3, out4, out5] = isOutputComplexImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8] = isOutputComplexImpl(obj) out1 = false; out2 = false; out3 = false; out4 = false; out5 = false; + out6 = false; + out7 = false; + out8 = false; end - function [out1, out2, out3, out4, out5] = isOutputFixedSizeImpl(obj) + function [out1, out2, out3, out4, out5, out6, out7, out8] = isOutputFixedSizeImpl(obj) out1 = true; out2 = true; out3 = true; out4 = true; out5 = true; + out6 = true; + out7 = true; + out8 = true; end function sts = getSampleTimeImpl(obj) diff --git a/extensions/client_matlab/+hl2ss/+ms/unpack_pv.m b/extensions/client_matlab/+hl2ss/+ms/unpack_pv.m index 6be8aeac7..680ebe090 100644 --- a/extensions/client_matlab/+hl2ss/+ms/unpack_pv.m +++ b/extensions/client_matlab/+hl2ss/+ms/unpack_pv.m @@ -12,6 +12,9 @@ 'timestamp', response.timestamp, ... 'image', response.image, ... 'intrinsics', response.intrinsics, ... + 'exposure', response.exposure, ... + 'imaging', response.imaging, ... + 'gains', response.gains, ... 'pose', pose); else data = struct('frame_index', response.frame_index, ... @@ -19,6 +22,9 @@ 'timestamp', zeros([1, 1], 'uint64'), ... 'image', zeros(image_size, 'uint8' ), ... 'intrinsics', zeros([4, 1], 'single'), ... + 'exposure', zeros([3, 1], 'uint64'), ... + 'imaging', zeros([4, 1], 'uint32'), ... + 'gains', zeros([5, 1], 'single'), ... 'pose', zeros([4, 4], 'single')); end end diff --git a/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_ahat.m b/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_ahat.m index 5d3e5be39..2192443ec 100644 --- a/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_ahat.m +++ b/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_ahat.m @@ -7,21 +7,23 @@ pose = zeros([4, 4], 'single'); end - data = struct('frame_index', response.frame_index, ... - 'status', response.status, ... - 'timestamp', response.timestamp, ... - 'depth', response.depth, ... - 'ab', response.ab, ... - 'pose', pose); + data = struct('frame_index', response.frame_index, ... + 'status', response.status, ... + 'timestamp', response.timestamp, ... + 'depth', response.depth, ... + 'ab', response.ab, ... + 'sensor_ticks', response.sensor_ticks, ... + 'pose', pose); else image_size = [hl2ss.parameters_rm_depth_ahat.HEIGHT, hl2ss.parameters_rm_depth_ahat.WIDTH]; zero_image = zeros(image_size, 'uint16'); - data = struct('frame_index', response.frame_index, ... - 'status', response.status, ... - 'timestamp', zeros([1, 1], 'uint64'), ... - 'depth', zero_image, ... - 'ab', zero_image, ... - 'pose', zeros([4, 4], 'single')); + data = struct('frame_index', response.frame_index, ... + 'status', response.status, ... + 'timestamp', zeros([1, 1], 'uint64'), ... + 'depth', zero_image, ... + 'ab', zero_image, ... + 'sensor_ticks', zeros([1, 1], 'uint64'), ... + 'pose', zeros([4, 4], 'single')); end end diff --git a/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_longthrow.m b/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_longthrow.m index db2a27fe4..2676515f3 100644 --- a/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_longthrow.m +++ b/extensions/client_matlab/+hl2ss/+ms/unpack_rm_depth_longthrow.m @@ -7,21 +7,23 @@ pose = zeros([4, 4], 'single'); end - data = struct('frame_index', response.frame_index, ... - 'status', response.status, ... - 'timestamp', response.timestamp, ... - 'depth', response.depth, ... - 'ab', response.ab, ... - 'pose', pose); + data = struct('frame_index', response.frame_index, ... + 'status', response.status, ... + 'timestamp', response.timestamp, ... + 'depth', response.depth, ... + 'ab', response.ab, ... + 'sensor_ticks', response.sensor_ticks, ... + 'pose', pose); else image_size = [hl2ss.parameters_rm_depth_longthrow.HEIGHT, hl2ss.parameters_rm_depth_longthrow.WIDTH]; zero_image = zeros(image_size, 'uint16'); - data = struct('frame_index', response.frame_index, ... - 'status', response.status, ... - 'timestamp', zeros([1, 1], 'uint64'), ... - 'depth', zero_image, ... - 'ab', zero_image, ... - 'pose', zeros([4, 4], 'single')); + data = struct('frame_index', response.frame_index, ... + 'status', response.status, ... + 'timestamp', zeros([1, 1], 'uint64'), ... + 'depth', zero_image, ... + 'ab', zero_image, ... + 'sensor_ticks', zeros([1, 1], 'uint64'), ... + 'pose', zeros([4, 4], 'single')); end end diff --git a/extensions/client_matlab/+hl2ss/+ms/unpack_rm_vlc.m b/extensions/client_matlab/+hl2ss/+ms/unpack_rm_vlc.m index 54c256781..baab1f319 100644 --- a/extensions/client_matlab/+hl2ss/+ms/unpack_rm_vlc.m +++ b/extensions/client_matlab/+hl2ss/+ms/unpack_rm_vlc.m @@ -7,18 +7,24 @@ pose = zeros([4, 4], 'single'); end - data = struct('frame_index', response.frame_index, ... - 'status', response.status, ... - 'timestamp', response.timestamp, ... - 'image', response.image, ... - 'pose', pose); + data = struct('frame_index', response.frame_index, ... + 'status', response.status, ... + 'timestamp', response.timestamp, ... + 'image', response.image, ... + 'sensor_ticks', response.sensor_ticks, ... + 'exposure', response.exposure, ... + 'gain', response.gain, ... + 'pose', pose); else image_size = [hl2ss.parameters_rm_vlc.HEIGHT, hl2ss.parameters_rm_vlc.WIDTH]; - data = struct('frame_index', response.frame_index, ... - 'status', response.status, ... - 'timestamp', zeros([1, 1], 'uint64'), ... - 'image', zeros(image_size, 'uint8' ), ... - 'pose', zeros([4, 4], 'single')); + data = struct('frame_index', response.frame_index, ... + 'status', response.status, ... + 'timestamp', zeros([1, 1], 'uint64'), ... + 'image', zeros(image_size, 'uint8' ), ... + 'sensor_ticks', zeros([1, 1], 'uint64'), ... + 'exposure', zeros([1, 1], 'uint64'), ... + 'gain', zeros([1, 1], 'uint32'), ... + 'pose', zeros([4, 4], 'single')); end end diff --git a/extensions/client_matlab/+hl2ss/+mt/ipc_rc.m b/extensions/client_matlab/+hl2ss/+mt/ipc_rc.m index 8c755dd90..b6ff9c271 100644 --- a/extensions/client_matlab/+hl2ss/+mt/ipc_rc.m +++ b/extensions/client_matlab/+hl2ss/+mt/ipc_rc.m @@ -82,5 +82,33 @@ function set_pv_white_balance_value(self, value) function set_flat_mode(self, value) self.module('ipc_call', self.port, 'set_flat_mode', uint32(value)); end + + function set_rm_eye_selection(self, enable) + self.module('ipc_call', self.port, 'set_rm_eye_selection', uint32(enable)); + end + + function set_pv_desired_optimization(self, mode) + self.module('ipc_call', self.port, 'set_pv_desired_optimization', uint32(mode)); + end + + function set_pv_primary_use(self, mode) + self.module('ipc_call', self.port, 'set_pv_primary_use', uint32(mode)); + end + + function set_pv_optical_image_stabilization(self, mode) + self.module('ipc_call', self.port, 'set_pv_optical_image_stabilization', uint32(mode)); + end + + function set_pv_hdr_video(self, mode) + self.module('ipc_call', self.port, 'set_pv_hdr_video', uint32(mode)); + end + + function set_pv_regions_of_interest(self, clear, set, auto_exposure, auto_focus, bounds_normalized, type, weight, x, y, w, h) + self.module('ipc_call', self.port, 'set_pv_regions_of_interest', logical(clear), logical(set), logical(auto_exposure), logical(auto_focus), logical(bounds_normalized), uint32(type), uint32(weight), single(x), single(y), single(w), single(h)); + end + + function set_interface_priority(self, port, priority) + self.module('ipc_call', self.port, 'set_interface_priority', uint16(port), int32(priority)); + end end end diff --git a/extensions/client_matlab/+hl2ss/interface_priority.m b/extensions/client_matlab/+hl2ss/interface_priority.m new file mode 100644 index 000000000..b7981f82e --- /dev/null +++ b/extensions/client_matlab/+hl2ss/interface_priority.m @@ -0,0 +1,10 @@ + +classdef interface_priority +properties (Constant) + LOWEST = int32(-2); + BELOW_NORMAL = int32(-1); + NORMAL = int32(0); + ABOVE_NORMAL = int32(1); + HIGHEST = int32(2); +end +end diff --git a/extensions/client_matlab/+hl2ss/pv_capture_use.m b/extensions/client_matlab/+hl2ss/pv_capture_use.m new file mode 100644 index 000000000..8a8016d98 --- /dev/null +++ b/extensions/client_matlab/+hl2ss/pv_capture_use.m @@ -0,0 +1,8 @@ + +classdef pv_capture_use +properties (Constant) + NotSet = uint32(0); + Photo = uint32(1); + Video = uint32(2); +end +end diff --git a/extensions/client_matlab/+hl2ss/pv_focus_state.m b/extensions/client_matlab/+hl2ss/pv_focus_state.m new file mode 100644 index 000000000..8f11b3561 --- /dev/null +++ b/extensions/client_matlab/+hl2ss/pv_focus_state.m @@ -0,0 +1,10 @@ + +classdef pv_focus_state +properties (Constant) + UNINITIALIZED = uint32(0); + LOST = uint32(1); + SEARCHING = uint32(2); + FOCUSED = uint32(3); + FAILED = uint32(4); +end +end diff --git a/extensions/client_matlab/+hl2ss/pv_hdr_video_mode.m b/extensions/client_matlab/+hl2ss/pv_hdr_video_mode.m new file mode 100644 index 000000000..2412d3d94 --- /dev/null +++ b/extensions/client_matlab/+hl2ss/pv_hdr_video_mode.m @@ -0,0 +1,8 @@ + +classdef pv_hdr_video_mode +properties (Constant) + Off = uint32(0); + On = uint32(1); + Auto = uint32(2); +end +end diff --git a/extensions/client_matlab/+hl2ss/pv_media_capture_optimization.m b/extensions/client_matlab/+hl2ss/pv_media_capture_optimization.m new file mode 100644 index 000000000..f3aa9150e --- /dev/null +++ b/extensions/client_matlab/+hl2ss/pv_media_capture_optimization.m @@ -0,0 +1,12 @@ + +classdef pv_media_capture_optimization +properties (Constant) + Default = uint32(0); + Quality = uint32(1); + Latency = uint32(2); + Power = uint32(3); + LatencyThenQuality = uint32(4); + LatencyThenPower = uint32(5); + PowerAndQuality = uint32(6); +end +end diff --git a/extensions/client_matlab/+hl2ss/pv_optical_stabilization_mode.m b/extensions/client_matlab/+hl2ss/pv_optical_stabilization_mode.m new file mode 100644 index 000000000..14589a7e6 --- /dev/null +++ b/extensions/client_matlab/+hl2ss/pv_optical_stabilization_mode.m @@ -0,0 +1,7 @@ + +classdef pv_optical_stabilization_mode +properties (Constant) + Off = uint32(0); + On = uint32(1); +end +end diff --git a/extensions/client_matlab/+hl2ss/pv_region_of_interest_type.m b/extensions/client_matlab/+hl2ss/pv_region_of_interest_type.m new file mode 100644 index 000000000..da51a5eb0 --- /dev/null +++ b/extensions/client_matlab/+hl2ss/pv_region_of_interest_type.m @@ -0,0 +1,7 @@ + +classdef pv_region_of_interest_type +properties (Constant) + Unknown = uint32(0); + Face = uint32(1); +end +end diff --git a/extensions/client_matlab/hl2ss_matlab.cpp b/extensions/client_matlab/hl2ss_matlab.cpp index 0b69493c1..0863c2cb3 100644 --- a/extensions/client_matlab/hl2ss_matlab.cpp +++ b/extensions/client_matlab/hl2ss_matlab.cpp @@ -343,21 +343,30 @@ class MexFunction : public matlab::mex::Function void pack_rm_vlc(int64_t frame_index, int32_t status, hl2ss::packet* packet, matlab::mex::ArgumentList outputs) { - matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "image", "pose" }); + uint32_t size = hl2ss::parameters_rm_vlc::PIXELS * sizeof(uint8_t); - o[0]["frame_index"] = m_factory.createScalar(frame_index); - o[0]["status"] = m_factory.createScalar(status); + matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "image", "sensor_ticks", "exposure", "gain", "pose" }); + + o[0]["frame_index"] = m_factory.createScalar(frame_index); + o[0]["status"] = m_factory.createScalar(status); if (packet) { - o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); + o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); if (packet->payload) { - o[0]["image"] = unpack_payload(packet->payload.get(), 0, packet->sz_payload, { hl2ss::parameters_rm_vlc::HEIGHT, hl2ss::parameters_rm_vlc::WIDTH }); + uint8_t* image; + hl2ss::rm_vlc_metadata* metadata; + hl2ss::unpack_rm_vlc(packet->payload.get(), image, metadata); + + o[0]["image"] = unpack_payload(image, 0, size, { hl2ss::parameters_rm_vlc::HEIGHT, hl2ss::parameters_rm_vlc::WIDTH }); + o[0]["sensor_ticks"] = m_factory.createScalar(metadata->sensor_ticks); + o[0]["exposure"] = m_factory.createScalar(metadata->exposure); + o[0]["gain"] = m_factory.createScalar(metadata->gain); } if (packet->pose) { - o[0]["pose"] = unpack_pose(packet->pose.get()); + o[0]["pose"] = unpack_pose(packet->pose.get()); } } @@ -368,22 +377,28 @@ class MexFunction : public matlab::mex::Function { uint32_t size = hl2ss::parameters_rm_depth_ahat::PIXELS * sizeof(uint16_t); - matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "depth", "ab", "pose" }); + matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "depth", "ab", "sensor_ticks", "pose" }); - o[0]["frame_index"] = m_factory.createScalar(frame_index); - o[0]["status"] = m_factory.createScalar(status); + o[0]["frame_index"] = m_factory.createScalar(frame_index); + o[0]["status"] = m_factory.createScalar(status); if (packet) { - o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); + o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); if (packet->payload) { - o[0]["depth"] = unpack_payload(packet->payload.get(), 0, size, { hl2ss::parameters_rm_depth_ahat::HEIGHT, hl2ss::parameters_rm_depth_ahat::WIDTH }); - o[0]["ab"] = unpack_payload(packet->payload.get(), size, size, { hl2ss::parameters_rm_depth_ahat::HEIGHT, hl2ss::parameters_rm_depth_ahat::WIDTH }); + uint16_t* depth; + uint16_t* ab; + hl2ss::rm_depth_ahat_metadata* metadata; + hl2ss::unpack_rm_depth_ahat(packet->payload.get(), depth, ab, metadata); + + o[0]["depth"] = unpack_payload(depth, 0, size, { hl2ss::parameters_rm_depth_ahat::HEIGHT, hl2ss::parameters_rm_depth_ahat::WIDTH }); + o[0]["ab"] = unpack_payload(ab, 0, size, { hl2ss::parameters_rm_depth_ahat::HEIGHT, hl2ss::parameters_rm_depth_ahat::WIDTH }); + o[0]["sensor_ticks"] = m_factory.createScalar(metadata->sensor_ticks); } if (packet->pose) { - o[0]["pose"] = unpack_pose(packet->pose.get()); + o[0]["pose"] = unpack_pose(packet->pose.get()); } } @@ -394,21 +409,27 @@ class MexFunction : public matlab::mex::Function { uint32_t size = hl2ss::parameters_rm_depth_longthrow::PIXELS * sizeof(uint16_t); - matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "depth", "ab", "pose" }); + matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "depth", "ab", "sensor_ticks", "pose" }); - o[0]["frame_index"] = m_factory.createScalar(frame_index); - o[0]["status"] = m_factory.createScalar(status); + o[0]["frame_index"] = m_factory.createScalar(frame_index); + o[0]["status"] = m_factory.createScalar(status); if (packet) { - o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); + o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); if (packet->payload) { - o[0]["depth"] = unpack_payload(packet->payload.get(), 0, size, { hl2ss::parameters_rm_depth_longthrow::HEIGHT, hl2ss::parameters_rm_depth_longthrow::WIDTH }); - o[0]["ab"] = unpack_payload(packet->payload.get(), size, size, { hl2ss::parameters_rm_depth_longthrow::HEIGHT, hl2ss::parameters_rm_depth_longthrow::WIDTH }); + uint16_t* depth; + uint16_t* ab; + hl2ss::rm_depth_longthrow_metadata* metadata; + hl2ss::unpack_rm_depth_longthrow(packet->payload.get(), depth, ab, metadata); + + o[0]["depth"] = unpack_payload(depth, 0, size, { hl2ss::parameters_rm_depth_longthrow::HEIGHT, hl2ss::parameters_rm_depth_longthrow::WIDTH }); + o[0]["ab"] = unpack_payload(ab, 0, size, { hl2ss::parameters_rm_depth_longthrow::HEIGHT, hl2ss::parameters_rm_depth_longthrow::WIDTH }); + o[0]["sensor_ticks"] = m_factory.createScalar(metadata->sensor_ticks); } if (packet->pose) { - o[0]["pose"] = unpack_pose(packet->pose.get()); + o[0]["pose"] = unpack_pose(packet->pose.get()); } } @@ -467,7 +488,7 @@ class MexFunction : public matlab::mex::Function void pack_pv(int64_t frame_index, int32_t status, hl2ss::packet* packet, uint16_t width, uint16_t height, uint8_t channels, matlab::mex::ArgumentList outputs) { - matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "image", "intrinsics", "pose" }); + matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "frame_index", "status", "timestamp", "image", "intrinsics", "exposure", "imaging", "gains", "pose" }); o[0]["frame_index"] = m_factory.createScalar(frame_index); o[0]["status"] = m_factory.createScalar(status); @@ -477,9 +498,16 @@ class MexFunction : public matlab::mex::Function o[0]["timestamp"] = m_factory.createScalar(packet->timestamp); if (packet->payload) { - uint32_t image_size = packet->sz_payload - hl2ss::decoder_pv::K_SIZE; - o[0]["image"] = unpack_payload(packet->payload.get(), 0, image_size, { height, width, channels }); - o[0]["intrinsics"] = unpack_payload( packet->payload.get(), image_size, hl2ss::decoder_pv::K_SIZE, { 4 }); + uint32_t image_size = packet->sz_payload - hl2ss::decoder_pv::METADATA_SIZE; + uint8_t* image; + hl2ss::pv_metadata* metadata; + hl2ss::unpack_pv(packet->payload.get(), packet->sz_payload, image, metadata); + + o[0]["image"] = unpack_payload( image, 0, image_size, { height, width, channels }); + o[0]["intrinsics"] = unpack_payload( &metadata->f, 0, 4 * sizeof(float), { 4 }); + o[0]["exposure"] = unpack_payload(&metadata->exposure_time, 0, 3 * sizeof(uint64_t), { 3 }); + o[0]["imaging"] = unpack_payload(&metadata->lens_position, 0, 4 * sizeof(uint32_t), { 4 }); + o[0]["gains"] = unpack_payload (&metadata->iso_gains, 0, 5 * sizeof(float), { 5 }); } if (packet->pose) { @@ -922,7 +950,7 @@ class MexFunction : public matlab::mex::Function std::shared_ptr data = hl2ss::lnm::download_calibration_pv(host, port, width, height, framerate); - matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "focal_length", "principal_point", "radial_distortion", "tangential_distortion", "projection", "extrinsics" }); + matlab::data::StructArray o = m_factory.createStructArray({ 1 }, { "focal_length", "principal_point", "radial_distortion", "tangential_distortion", "projection", "extrinsics", "intrinsics_mf", "extrinsics_mf" }); o[0]["focal_length"] = to_typed_array(data->focal_length, { sizeof(data->focal_length) / sizeof(float) }); o[0]["principal_point"] = to_typed_array(data->principal_point, { sizeof(data->principal_point) / sizeof(float) }); @@ -930,6 +958,8 @@ class MexFunction : public matlab::mex::Function o[0]["tangential_distortion"] = to_typed_array(data->tangential_distortion, { sizeof(data->tangential_distortion) / sizeof(float) }); o[0]["projection"] = unpack_payload(&data->projection, 0, sizeof(data->projection), { 4, 4 }); o[0]["extrinsics"] = unpack_payload(&data->extrinsics, 0, sizeof(data->extrinsics), { 4, 4 }); + o[0]["intrinsics_mf"] = to_typed_array(data->intrinsics_mf, { sizeof(data->intrinsics_mf) / sizeof(float) }); + o[0]["extrinsics_mf"] = to_typed_array(data->extrinsics_mf, { sizeof(data->extrinsics_mf) / sizeof(float) }); outputs[0] = std::move(o); } @@ -1040,6 +1070,52 @@ class MexFunction : public matlab::mex::Function uint32_t value = get_argument(inputs); ipc_rc->set_flat_mode(value); } + else if (f == "set_rm_eye_selection") + { + uint32_t enable = get_argument(inputs); + ipc_rc->set_rm_eye_selection(enable); + } + else if (f == "set_pv_desired_optimization") + { + uint32_t mode = get_argument(inputs); + ipc_rc->set_pv_desired_optimization(mode); + } + else if (f == "set_pv_primary_use") + { + uint32_t mode = get_argument(inputs); + ipc_rc->set_pv_primary_use(mode); + } + else if (f == "set_pv_optical_image_stabilization") + { + uint32_t mode = get_argument(inputs); + ipc_rc->set_pv_optical_image_stabilization(mode); + } + else if (f == "set_pv_hdr_video") + { + uint32_t mode = get_argument(inputs); + ipc_rc->set_pv_hdr_video(mode); + } + else if (f == "set_pv_regions_of_interest") + { + bool clear = get_argument(inputs); + bool set = get_argument(inputs); + bool auto_exposure = get_argument(inputs); + bool auto_focus = get_argument(inputs); + bool bounds_normalized = get_argument(inputs); + uint32_t type = get_argument(inputs); + uint32_t weight = get_argument(inputs); + float x = get_argument(inputs); + float y = get_argument(inputs); + float w = get_argument(inputs); + float h = get_argument(inputs); + ipc_rc->set_pv_regions_of_interest(clear, set, auto_exposure, auto_focus, bounds_normalized, type, weight, x, y, w, h); + } + else if (f == "set_interface_priority") + { + uint16_t port = get_argument(inputs); + int32_t priority = get_argument(inputs); + ipc_rc->set_interface_priority(port, priority); + } else { throw std::runtime_error("Unknown method"); @@ -1066,10 +1142,10 @@ class MexFunction : public matlab::mex::Function { switch (get_field_scalar(p[i]["type"])) { - case hl2ss::sm_volume_type::Box: volumes.add_box(get_field_vector_3(p[i]["center"]), get_field_vector_3(p[i]["extents"])); break; - case hl2ss::sm_volume_type::Frustum: volumes.add_frustum(get_field_vector_4(p[i]["near"]), get_field_vector_4(p[i]["far"]), get_field_vector_4(p[i]["right"]), get_field_vector_4(p[i]["left"]), get_field_vector_4(p[i]["top"]), get_field_vector_4(p[i]["bottom"])); break; - case hl2ss::sm_volume_type::OrientedBox: volumes.add_oriented_box(get_field_vector_3(p[i]["center"]), get_field_vector_3(p[i]["extents"]), get_field_vector_4(p[i]["orientation"])); break; - case hl2ss::sm_volume_type::Sphere: volumes.add_sphere(get_field_vector_3(p[i]["center"]), get_field_scalar(p[i]["radius"])); break; + case hl2ss::sm_volume_type::Box: volumes.add_box({ get_field_vector_3(p[i]["center"]), get_field_vector_3(p[i]["extents"]) }); break; + case hl2ss::sm_volume_type::Frustum: volumes.add_frustum({ get_field_vector_4(p[i]["near"]), get_field_vector_4(p[i]["far"]), get_field_vector_4(p[i]["right"]), get_field_vector_4(p[i]["left"]), get_field_vector_4(p[i]["top"]), get_field_vector_4(p[i]["bottom"]) }); break; + case hl2ss::sm_volume_type::OrientedBox: volumes.add_oriented_box({ get_field_vector_3(p[i]["center"]), get_field_vector_3(p[i]["extents"]), get_field_vector_4(p[i]["orientation"]) }); break; + case hl2ss::sm_volume_type::Sphere: volumes.add_sphere({ get_field_vector_3(p[i]["center"]), get_field_scalar(p[i]["radius"]) }); break; default: break; } } diff --git a/extensions/client_matlab/viewer/client_ipc_rc.m b/extensions/client_matlab/viewer/client_ipc_rc.m index db8cbe48b..a22fe5e49 100644 --- a/extensions/client_matlab/viewer/client_ipc_rc.m +++ b/extensions/client_matlab/viewer/client_ipc_rc.m @@ -26,8 +26,14 @@ client.set_pv_video_temporal_denoising(hl2ss.pv_video_temporal_denoising_mode.Off); client.set_pv_white_balance_preset(hl2ss.pv_color_temperature_preset.Auto); client.set_pv_white_balance_value(hl2ss.pv_white_balance_value.Min); -client.set_flat_mode(0) - +client.set_flat_mode(false) +client.set_rm_eye_selection(false); +client.set_pv_desired_optimization(hl2ss.pv_media_capture_optimization.LatencyThenPower); +client.set_pv_primary_use(hl2ss.pv_capture_use.Video); +client.set_pv_optical_image_stabilization(hl2ss.pv_optical_stabilization_mode.On); +client.set_pv_hdr_video(hl2ss.pv_hdr_video_mode.Off); +client.set_pv_regions_of_interest(true, true, true, true, true, hl2ss.pv_region_of_interest_type.Unknown, 100, 0, 0, 1, 1); +client.set_interface_priority(hl2ss.stream_port.PERSONAL_VIDEO, hl2ss.interface_priority.ABOVE_NORMAL); catch ME disp(ME.message); end diff --git a/extensions/client_matlab/viewer_simulink.slx b/extensions/client_matlab/viewer_simulink.slx index a2778f5e1..7bd7ad8fc 100644 Binary files a/extensions/client_matlab/viewer_simulink.slx and b/extensions/client_matlab/viewer_simulink.slx differ diff --git a/hl2ss/hl2ss/Package.appxmanifest b/hl2ss/hl2ss/Package.appxmanifest index 463a32c2f..18274e54c 100644 --- a/hl2ss/hl2ss/Package.appxmanifest +++ b/hl2ss/hl2ss/Package.appxmanifest @@ -9,7 +9,7 @@ + Version="1.0.33.0" /> hl2ss diff --git a/hl2ss/hl2ss/custom_media_sink.cpp b/hl2ss/hl2ss/custom_media_sink.cpp index c22dcb706..8aac22be8 100644 --- a/hl2ss/hl2ss/custom_media_sink.cpp +++ b/hl2ss/hl2ss/custom_media_sink.cpp @@ -50,7 +50,7 @@ DWORD CustomMediaSink::GetStreamId(IMFStreamSink* stream) // OK CustomStreamSink* CustomMediaSink::GetStreamSinkById(DWORD dwStreamSinkIdentifier) { - for (auto stream : m_streams) { if (GetStreamId(stream) == dwStreamSinkIdentifier) { return stream; } } + for (auto const& stream : m_streams) { if (GetStreamId(stream) == dwStreamSinkIdentifier) { return stream; } } return NULL; } @@ -58,14 +58,14 @@ CustomStreamSink* CustomMediaSink::GetStreamSinkById(DWORD dwStreamSinkIdentifie CustomStreamSink* CustomMediaSink::GetStreamSinkByIndex(DWORD dwIndex) { DWORD idx = 0; - for (auto stream : m_streams) { if (idx++ == dwIndex) { return stream; } } + for (auto const& stream : m_streams) { if (idx++ == dwIndex) { return stream; } } return NULL; } // OK void CustomMediaSink::CleanupStreams() { - for (auto stream : m_streams) { stream->Release(); } + for (auto const& stream : m_streams) { stream->Release(); } m_streams.clear(); } @@ -207,7 +207,7 @@ HRESULT CustomMediaSink::Shutdown() { CriticalSection cs(&m_critSec); if (m_isShutdown) { return MF_E_SHUTDOWN; } - for (auto stream : m_streams) { stream->Shutdown(); } // TODO: Error Handling + for (auto const& stream : m_streams) { stream->Shutdown(); } // TODO: Error Handling //SafeRelease(&m_pClock); CleanupStreams(); m_isShutdown = true; @@ -226,7 +226,7 @@ HRESULT CustomMediaSink::OnClockStart(MFTIME hnsSystemTime, LONGLONG llClockStar { CriticalSection cs(&m_critSec); if (m_isShutdown) { return MF_E_SHUTDOWN; } - for (auto stream : m_streams) { stream->Start(hnsSystemTime, llClockStartOffset); } // TODO: Error Handling + for (auto const& stream : m_streams) { stream->Start(hnsSystemTime, llClockStartOffset); } // TODO: Error Handling return S_OK; } @@ -235,7 +235,7 @@ HRESULT CustomMediaSink::OnClockStop(MFTIME hnsSystemTime) { CriticalSection cs(&m_critSec); if (m_isShutdown) { return MF_E_SHUTDOWN; } - for (auto stream : m_streams) { stream->Stop(hnsSystemTime); } // TODO: Error Handling + for (auto const& stream : m_streams) { stream->Stop(hnsSystemTime); } // TODO: Error Handling return S_OK; } @@ -244,7 +244,7 @@ HRESULT CustomMediaSink::OnClockPause(MFTIME hnsSystemTime) { CriticalSection cs(&m_critSec); if (m_isShutdown) { return MF_E_SHUTDOWN; } - for (auto stream : m_streams) { stream->Pause(hnsSystemTime); } // TODO: Error Handling + for (auto const& stream : m_streams) { stream->Pause(hnsSystemTime); } // TODO: Error Handling return S_OK; } @@ -253,7 +253,7 @@ HRESULT CustomMediaSink::OnClockRestart(MFTIME hnsSystemTime) { CriticalSection cs(&m_critSec); if (m_isShutdown) { return MF_E_SHUTDOWN; } - for (auto stream : m_streams) { stream->Restart(hnsSystemTime); } // TODO: Error Handling + for (auto const& stream : m_streams) { stream->Restart(hnsSystemTime); } // TODO: Error Handling return S_OK; } @@ -262,6 +262,6 @@ HRESULT CustomMediaSink::OnClockSetRate(MFTIME hnsSystemTime, float flRate) { CriticalSection cs(&m_critSec); if (m_isShutdown) { return MF_E_SHUTDOWN; } - for (auto stream : m_streams) { stream->SetRate(hnsSystemTime, flRate); } // TODO: Error Handling + for (auto const& stream : m_streams) { stream->SetRate(hnsSystemTime, flRate); } // TODO: Error Handling return S_OK; } diff --git a/hl2ss/hl2ss/custom_sink_writers.cpp b/hl2ss/hl2ss/custom_sink_writers.cpp index 1fbba2ee2..65508bbcd 100644 --- a/hl2ss/hl2ss/custom_sink_writers.cpp +++ b/hl2ss/hl2ss/custom_sink_writers.cpp @@ -52,12 +52,14 @@ static uint32_t ComputeStride(uint32_t width) // OK static void TranslateEncoderOptions(std::vector const& options, IMFAttributes **pEncoderAttr) { - MFCreateAttributes(pEncoderAttr, (UINT32)(options.size() / 2)); + size_t size = options.size() & ~1ULL; - for (int i = 0; i < (int)(options.size() / 2); ++i) + MFCreateAttributes(pEncoderAttr, (UINT32)size); + + for (int i = 0; i < (int)size; i += 2) { - uint64_t option = options[(2 * i)]; - uint64_t value = options[(2 * i) + 1]; + uint64_t option = options[i]; + uint64_t value = options[i + 1]; if (option >= (sizeof(g_AVLUT) / sizeof(AVOption))) { continue; } diff --git a/hl2ss/hl2ss/extended_audio.cpp b/hl2ss/hl2ss/extended_audio.cpp index 8b6103529..e867096b4 100644 --- a/hl2ss/hl2ss/extended_audio.cpp +++ b/hl2ss/hl2ss/extended_audio.cpp @@ -62,11 +62,11 @@ static bool ExtendedAudio_FindAudioSource(MediaCapture const& mediaCapture, Medi for (auto const& frameSource : mediaCapture.FrameSources()) { - auto source = frameSource.Value(); + auto const& source = frameSource.Value(); if (source.Info().MediaStreamType() != MediaStreamType::Audio) { continue; } for (auto const& format : source.SupportedFormats()) { - auto aep = format.AudioEncodingProperties(); + auto const& aep = format.AudioEncodingProperties(); if (aep.SampleRate() != 48000) { continue; } AudioSubtype subtype; bool ok = ExtendedAudio_ParseSubtype(aep.Subtype(), subtype); diff --git a/hl2ss/hl2ss/extended_execution.cpp b/hl2ss/hl2ss/extended_execution.cpp index 1ce785f96..2881eb9ea 100644 --- a/hl2ss/hl2ss/extended_execution.cpp +++ b/hl2ss/hl2ss/extended_execution.cpp @@ -1,8 +1,8 @@ +#include #include "log.h" #include - #include #include #include @@ -23,6 +23,8 @@ static wchar_t const* g_flat_name = L"flat_mode.cfg"; static ExtendedExecutionForegroundSession g_eefs = nullptr; static bool g_status = false; +static std::atomic g_interface_priority[32]; + //----------------------------------------------------------------------------- // Functions //----------------------------------------------------------------------------- @@ -94,3 +96,17 @@ bool ExtendedExecution_GetFlatMode() return false; } + +// OK +void ExtendedExecution_SetInterfacePriority(uint32_t id, int32_t priority) +{ + if (id >= 32) { return; } + if ((priority < -2) || (priority > 2)) { return; } + g_interface_priority[id] = priority; +} + +// OK +int32_t ExtendedExecution_GetInterfacePriority(uint32_t id) +{ + return g_interface_priority[id]; +} diff --git a/hl2ss/hl2ss/extended_execution.h b/hl2ss/hl2ss/extended_execution.h index aedd6bca7..551464557 100644 --- a/hl2ss/hl2ss/extended_execution.h +++ b/hl2ss/hl2ss/extended_execution.h @@ -5,3 +5,5 @@ void ExtendedExecution_Request(); bool ExtendedExecution_Status(); void ExtendedExecution_SetFlatMode(bool flat); bool ExtendedExecution_GetFlatMode(); +void ExtendedExecution_SetInterfacePriority(uint32_t id, int32_t priority); +int32_t ExtendedExecution_GetInterfacePriority(uint32_t id); diff --git a/hl2ss/hl2ss/extended_eye_tracking.cpp b/hl2ss/hl2ss/extended_eye_tracking.cpp index e211359aa..755a836dc 100644 --- a/hl2ss/hl2ss/extended_eye_tracking.cpp +++ b/hl2ss/hl2ss/extended_eye_tracking.cpp @@ -68,7 +68,7 @@ void ExtendedEyeTracking_QueryCapabilities() ShowMessage("IsRestrictedModeSupported: %d", g_tracker.IsRestrictedModeSupported()); ShowMessage("IsVergenceDistanceSupported: %d", g_tracker.IsVergenceDistanceSupported()); ShowMessage("SupportedTargetFrameRates"); - for (auto stfr : g_tracker.SupportedTargetFrameRates()) { ShowMessage("%d FPS", stfr.FramesPerSecond()); } + for (auto const& stfr : g_tracker.SupportedTargetFrameRates()) { ShowMessage("%d FPS", stfr.FramesPerSecond()); } } // OK @@ -80,7 +80,7 @@ SpatialLocator ExtendedEyeTracking_CreateLocator() // OK void ExtendedEyeTracking_SetTargetFrameRate(int index) { - auto stfrs = g_tracker.SupportedTargetFrameRates(); + auto const& stfrs = g_tracker.SupportedTargetFrameRates(); g_tracker.SetTargetFrameRate(stfrs.GetAt(index)); } diff --git a/hl2ss/hl2ss/extended_video.cpp b/hl2ss/hl2ss/extended_video.cpp index 9694eac66..32b526ac1 100644 --- a/hl2ss/hl2ss/extended_video.cpp +++ b/hl2ss/hl2ss/extended_video.cpp @@ -58,16 +58,16 @@ void ExtendedVideo_QueryDevices(winrt::hstring& out) JsonObject root = JsonObject(); for (uint32_t i = 0; i < ids.size(); ++i) { - auto sourceGroup = MediaFrameSourceGroup::FromIdAsync(ids[i]).get(); + auto const& sourceGroup = MediaFrameSourceGroup::FromIdAsync(ids[i]).get(); JsonObject jsourceinfos = JsonObject(); uint32_t sourceInfo = 0; - for (auto source : sourceGroup.SourceInfos()) + for (auto const& source : sourceGroup.SourceInfos()) { JsonObject jvpmd = JsonObject(); uint32_t description = 0; - for (auto md : source.VideoProfileMediaDescription()) + for (auto const& md : source.VideoProfileMediaDescription()) { JsonObject jmd = JsonObject(); jmd.Insert(L"Width", JsonValue::CreateNumberValue(md.Width())); @@ -90,12 +90,12 @@ void ExtendedVideo_QueryDevices(winrt::hstring& out) JsonObject jvideoprofiles = JsonObject(); uint32_t profileIndex = 0; - for (auto profile : MediaCapture::FindAllVideoProfiles(sourceGroup.Id())) + for (auto const& profile : MediaCapture::FindAllVideoProfiles(sourceGroup.Id())) { JsonObject jdescription = JsonObject(); uint32_t description = 0; - for (auto md : profile.SupportedRecordMediaDescription()) + for (auto const& md : profile.SupportedRecordMediaDescription()) { JsonObject jmd = JsonObject(); jmd.Insert(L"Width", JsonValue::CreateNumberValue(md.Width())); @@ -134,11 +134,11 @@ static bool ExtendedVideo_FindMediaSourceGroup(uint32_t indexGroup, uint32_t ind if (indexGroup >= ids.size()) { return false; } sourceGroup = MediaFrameSourceGroup::FromIdAsync(ids[indexGroup]).get(); - auto sources = sourceGroup.SourceInfos(); + auto const& sources = sourceGroup.SourceInfos(); if (indexSource >= sources.Size()) { return false; } sourceId = sources.GetAt(indexSource).Id(); - auto profiles = MediaCapture::FindAllVideoProfiles(sourceGroup.Id()); + auto const& profiles = MediaCapture::FindAllVideoProfiles(sourceGroup.Id()); if (profiles.Size() <= 0) { profile = nullptr; @@ -148,7 +148,7 @@ static bool ExtendedVideo_FindMediaSourceGroup(uint32_t indexGroup, uint32_t ind { if (indexProfile >= profiles.Size()) { return false; } profile = profiles.GetAt(indexProfile); - auto descriptions = profile.SupportedRecordMediaDescription(); + auto const& descriptions = profile.SupportedRecordMediaDescription(); if (descriptions.Size() <= 0) { return false; } description = descriptions.GetAt(0); } @@ -159,7 +159,7 @@ static bool ExtendedVideo_FindMediaSourceGroup(uint32_t indexGroup, uint32_t ind // OK static bool ExtendedVideo_FindVideoSource(MediaCapture const& mediaCapture, winrt::hstring const& sourceId, MediaFrameSource& videoSource) { - auto sources = mediaCapture.FrameSources(); + auto const& sources = mediaCapture.FrameSources(); if (!sources.HasKey(sourceId)) { return false; } videoSource = sources.Lookup(sourceId); return true; diff --git a/hl2ss/hl2ss/hl2ss.vcxproj b/hl2ss/hl2ss/hl2ss.vcxproj index d2a1b038b..c2a2feaf3 100644 --- a/hl2ss/hl2ss/hl2ss.vcxproj +++ b/hl2ss/hl2ss/hl2ss.vcxproj @@ -89,7 +89,7 @@ Always arm64 0 - 709C677708481A32A0540EC66BBEF7C9F029FCB8 + D98E441A0E47A44188C1A8D182A165044B5EFA8A diff --git a/hl2ss/hl2ss/holographic_space.cpp b/hl2ss/hl2ss/holographic_space.cpp index 37aeb6c7a..f92d4e51b 100644 --- a/hl2ss/hl2ss/holographic_space.cpp +++ b/hl2ss/hl2ss/holographic_space.cpp @@ -140,7 +140,7 @@ void HolographicSpace_Clear() for (HolographicCameraPose const& pose : g_frame.CurrentPrediction().CameraPoses()) { - auto rp = g_frame.GetRenderingParameters(pose); + auto const& rp = g_frame.GetRenderingParameters(pose); rp.Direct3D11BackBuffer().as()->GetInterface(IID_PPV_ARGS(&d3dBackBuffer)); g_context->CopyResource(d3dBackBuffer.Get(), g_enable_marker ? g_texture_marker : g_texture_empty); } diff --git a/hl2ss/hl2ss/ipc_rc.cpp b/hl2ss/hl2ss/ipc_rc.cpp index 17c1d0876..d0358d76b 100644 --- a/hl2ss/hl2ss/ipc_rc.cpp +++ b/hl2ss/hl2ss/ipc_rc.cpp @@ -5,8 +5,10 @@ #include "holographic_space.h" #include "personal_video.h" #include "extended_execution.h" +#include "research_mode.h" #include "timestamps.h" #include "nfo.h" +#include "types.h" //----------------------------------------------------------------------------- // Global Variables @@ -307,6 +309,166 @@ static void RC_MSG_SetFlatMode(SOCKET clientsocket) ExtendedExecution_SetFlatMode(mode != 0); } +// OK +static void RC_MSG_SetRMEyeSelection(SOCKET clientsocket) +{ + bool ok; + uint32_t enable; + + ok = recv_u32(clientsocket, enable); + if (!ok) + { + RC_TransferError(); + return; + } + + ResearchMode_SetEyeSelection(enable != 0); +} + +// OK +static void RC_MSG_SetPVDesiredOptimization(SOCKET clientsocket) +{ + bool ok; + uint32_t mode; + + ok = recv_u32(clientsocket, mode); + if (!ok) + { + RC_TransferError(); + return; + } + + PersonalVideo_SetDesiredOptimization(mode); +} + +// OK +static void RC_MSG_SetPVPrimaryUse(SOCKET clientsocket) +{ + bool ok; + uint32_t mode; + + ok = recv_u32(clientsocket, mode); + if (!ok) + { + RC_TransferError(); + return; + } + + PersonalVideo_SetPrimaryUse(mode); +} + +// OK +static void RC_MSG_SetPVOpticalImageStabilization(SOCKET clientsocket) +{ + bool ok; + uint32_t mode; + + ok = recv_u32(clientsocket, mode); + if (!ok) + { + RC_TransferError(); + return; + } + + PersonalVideo_SetOpticalImageStabilization(mode); +} + +// OK +static void RC_MSG_SetPVHdrVideo(SOCKET clientsocket) +{ + bool ok; + uint32_t mode; + + ok = recv_u32(clientsocket, mode); + if (!ok) + { + RC_TransferError(); + return; + } + + PersonalVideo_SetHdrVideo(mode); +} + +// OK +static void RC_MSG_SetRegionsOfInterest(SOCKET clientsocket) +{ + bool ok; + uint32_t mode; + float x; + float y; + float w; + float h; + + ok = recv_u32(clientsocket, mode); + if (!ok) + { + RC_TransferError(); + return; + } + + ok = recv_u32(clientsocket, (uint32_t&)x); + if (!ok) + { + RC_TransferError(); + return; + } + + ok = recv_u32(clientsocket, (uint32_t&)y); + if (!ok) + { + RC_TransferError(); + return; + } + + ok = recv_u32(clientsocket, (uint32_t&)w); + if (!ok) + { + RC_TransferError(); + return; + } + + ok = recv_u32(clientsocket, (uint32_t&)h); + if (!ok) + { + RC_TransferError(); + return; + } + + bool clear = bit_test( mode, 12); + bool set = bit_test( mode, 11); + bool auto_exposure = bit_test( mode, 10); + bool auto_focus = bit_test( mode, 9); + bool bounds_normalized = bit_test( mode, 8); + uint32_t type = bit_field(mode, 7, 0x01); + uint32_t weight = bit_field(mode, 0, 0x7F); + + PersonalVideo_SetRegionsOfInterest(clear, set, auto_exposure, auto_focus, bounds_normalized, x, y, w, h, type, weight); +} + +// OK +static void RC_MSG_SetInterfacePriority(SOCKET clientsocket) +{ + bool ok; + uint32_t id; + uint32_t priority; + + ok = recv_u32(clientsocket, id); + if (!ok) + { + RC_TransferError(); + return; + } + + ok = recv_u32(clientsocket, priority); + if (!ok) + { + RC_TransferError(); + return; + } + + ExtendedExecution_SetInterfacePriority(id - PORT_NUMBER_BASE, priority); +} + // OK static void RC_Dispatch(SOCKET clientsocket) { @@ -322,20 +484,27 @@ static void RC_Dispatch(SOCKET clientsocket) switch (state) { - case 0x00: RC_MSG_GetApplicationVersion(clientsocket); break; - case 0x01: RC_MSG_GetUTCOffset(clientsocket); break; - case 0x02: RC_MSG_SetHSMarkerState(clientsocket); break; - case 0x03: RC_MSG_GetPVSubsystemStatus(clientsocket); break; - case 0x04: RC_MSG_SetPVFocus(clientsocket); break; - case 0x05: RC_MSG_SetPVVideoTemporalDenoising(clientsocket); break; - case 0x06: RC_MSG_SetPVWhiteBalancePreset(clientsocket); break; - case 0x07: RC_MSG_SetPVWhiteBalanceValue(clientsocket); break; - case 0x08: RC_MSG_SetPVExposure(clientsocket); break; - case 0x09: RC_MSG_SetPVExposurePriorityVideo(clientsocket); break; - case 0x0A: RC_MSG_SetPVIsoSpeed(clientsocket); break; - case 0x0B: RC_MSG_SetPVBacklightCompensation(clientsocket); break; - case 0x0C: RC_MSG_SetPVSceneMode(clientsocket); break; - case 0x0D: RC_MSG_SetFlatMode(clientsocket); break; + case 0x00: RC_MSG_GetApplicationVersion(clientsocket); break; + case 0x01: RC_MSG_GetUTCOffset(clientsocket); break; + case 0x02: RC_MSG_SetHSMarkerState(clientsocket); break; + case 0x03: RC_MSG_GetPVSubsystemStatus(clientsocket); break; + case 0x04: RC_MSG_SetPVFocus(clientsocket); break; + case 0x05: RC_MSG_SetPVVideoTemporalDenoising(clientsocket); break; + case 0x06: RC_MSG_SetPVWhiteBalancePreset(clientsocket); break; + case 0x07: RC_MSG_SetPVWhiteBalanceValue(clientsocket); break; + case 0x08: RC_MSG_SetPVExposure(clientsocket); break; + case 0x09: RC_MSG_SetPVExposurePriorityVideo(clientsocket); break; + case 0x0A: RC_MSG_SetPVIsoSpeed(clientsocket); break; + case 0x0B: RC_MSG_SetPVBacklightCompensation(clientsocket); break; + case 0x0C: RC_MSG_SetPVSceneMode(clientsocket); break; + case 0x0D: RC_MSG_SetFlatMode(clientsocket); break; + case 0x0E: RC_MSG_SetRMEyeSelection(clientsocket); break; + case 0x0F: RC_MSG_SetPVDesiredOptimization(clientsocket); break; + case 0x10: RC_MSG_SetPVPrimaryUse(clientsocket); break; + case 0x11: RC_MSG_SetPVOpticalImageStabilization(clientsocket); break; + case 0x12: RC_MSG_SetPVHdrVideo(clientsocket); break; + case 0x13: RC_MSG_SetRegionsOfInterest(clientsocket); break; + case 0x14: RC_MSG_SetInterfacePriority(clientsocket); break; default: RC_TransferError(); return; diff --git a/hl2ss/hl2ss/ipc_su.cpp b/hl2ss/hl2ss/ipc_su.cpp index c9185be5d..439709d18 100644 --- a/hl2ss/hl2ss/ipc_su.cpp +++ b/hl2ss/hl2ss/ipc_su.cpp @@ -69,7 +69,7 @@ static void SU_TransferMeshes(SOCKET clientsocket, std::vectorGetVertexCount(); count_triangle_indices = mesh->GetTriangleIndexCount(); diff --git a/hl2ss/hl2ss/locator.cpp b/hl2ss/hl2ss/locator.cpp index 820689abe..a8df6af68 100644 --- a/hl2ss/hl2ss/locator.cpp +++ b/hl2ss/hl2ss/locator.cpp @@ -48,14 +48,14 @@ void Locator_Initialize() // OK float4x4 Locator_Locate(PerceptionTimestamp const& timestamp, SpatialLocator const& locator, SpatialCoordinateSystem const& world) { - auto location = locator.TryLocateAtTimestamp(timestamp, world); + auto const& location = locator.TryLocateAtTimestamp(timestamp, world); return location ? (make_float4x4_from_quaternion(location.Orientation()) * make_float4x4_translation(location.Position())) : float4x4(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); } // OK float4x4 Locator_GetTransformTo(SpatialCoordinateSystem const& src, SpatialCoordinateSystem const& dst) { - auto location = src.TryGetTransformTo(dst); + auto const& location = src.TryGetTransformTo(dst); return location ? location.Value() : float4x4(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); } diff --git a/hl2ss/hl2ss/main.cpp b/hl2ss/hl2ss/main.cpp index 3debae80a..322ca313a 100644 --- a/hl2ss/hl2ss/main.cpp +++ b/hl2ss/hl2ss/main.cpp @@ -108,7 +108,7 @@ struct App : winrt::implements void Run() { - auto window = CoreWindow::GetForCurrentThread(); + auto const& window = CoreWindow::GetForCurrentThread(); window.Activate(); while (!m_windowClosed) diff --git a/hl2ss/hl2ss/nfo.cpp b/hl2ss/hl2ss/nfo.cpp index ebd5cdd6d..2fbc10441 100644 --- a/hl2ss/hl2ss/nfo.cpp +++ b/hl2ss/hl2ss/nfo.cpp @@ -1,6 +1,7 @@ #include #include +#include "nfo.h" #include "log.h" #include @@ -32,7 +33,7 @@ void GetApplicationVersion(uint16_t data[4]) // OK void GetLocalIPv4Address(std::vector &address) { - for (auto hostname : NetworkInformation::GetHostNames()) + for (auto const& hostname : NetworkInformation::GetHostNames()) { if (hostname.Type() == HostNameType::Ipv4) { @@ -69,13 +70,93 @@ void PrintSystemInfo() ShowMessage("System Affinity Mask: %llX", sam); } +// OK +static void PrintVariant(winrt::hstring const& name, winrt::Windows::Foundation::IInspectable const& value) +{ + if (!value) + { + ShowMessage(L"%s : ? -> [NULL]", name.c_str()); + return; + } + + winrt::hstring type = winrt::get_class_name(value); + winrt::hstring text = L"[?]"; + + if (type == L"Windows.Foundation.IReferenceArray`1") // blob + { + auto const& w = value.try_as>(); + if (w) + { + auto const& m = w.Value(); + text = L"{" + winrt::to_hstring(m.size()) + L", ["; + for (auto a = m.begin(); a != m.end(); a++) { text = text + winrt::hstring(a != m.begin() ? L", " : L"") + winrt::to_hstring(*a); } + text = text + L"]}"; + }; + } + if (type == L"Windows.Foundation.IReference`1") + { + auto const& w = value.try_as(); + if (w) { text = w.value() ? L"true" : L"false"; } + } + if (type == L"Windows.Foundation.IReference`1") + { + auto const& w = value.try_as(); + if (w) { text = winrt::to_hstring(w.value()); } + } + if (type == L"Windows.Foundation.IReference`1") + { + auto const& w = value.try_as(); + if (w) { text = winrt::to_hstring(w.value()); } + } + if (type == L"Windows.Foundation.IReference`1") + { + auto const& w = value.try_as(); + if (w) { text = winrt::to_hstring(w.value()); } + } + if (type == L"Windows.Foundation.IReference`1") + { + auto const& w = value.try_as(); + if (w) { text = w.value(); } + } + if (type == L"Windows.Foundation.Collections.IMapView`2") + { + auto const& w = value.try_as>(); + if (w) + { + ShowMessage(L"SET %s BEGIN", name.c_str()); + PrintProperties(w); + ShowMessage(L"SET %s END", name.c_str()); + } + } + + ShowMessage(L"%s : %s -> %s", name.c_str(), type.c_str(), text.c_str()); +} + +// OK +void PrintProperties(winrt::Windows::Foundation::Collections::IMapView const& p) +{ + if (p) { for (auto const& kv : p) { PrintVariant(kv.Key(), kv.Value()); } } +} + +// OK +void PrintProperties(winrt::Windows::Foundation::Collections::IMapView const& p) +{ + if (p) { for (auto const& kv : p) { PrintVariant(winrt::to_hstring(kv.Key()), kv.Value()); } } +} + +// OK +void PrintProperties(winrt::Windows::Foundation::Collections::IPropertySet const& p) +{ + if (p) { for (auto const& kv : p) { PrintVariant(kv.Key(), kv.Value()); } } +} + // OK void PrintDeviceList(DeviceClass dc) { - auto list = DeviceInformation::FindAllAsync(dc).get(); + auto const& list = DeviceInformation::FindAllAsync(dc).get(); ShowMessage("Items of DeviceClass %d", (int)dc); - for (auto item : list) + for (auto const& item : list) { ShowMessage(L"Name: %s", item.Name().c_str()); ShowMessage(L"Id: %s", item.Id().c_str()); @@ -84,7 +165,7 @@ void PrintDeviceList(DeviceClass dc) ShowMessage("IsEnabled: %d", (int)item.IsEnabled()); ShowMessage("Kind: %d", (int)item.Kind()); - auto el = item.EnclosureLocation(); + auto const& el = item.EnclosureLocation(); if (el) { @@ -94,7 +175,7 @@ void PrintDeviceList(DeviceClass dc) ShowMessage("RotationAngleInDegreesClockwise: %d", (int)el.RotationAngleInDegreesClockwise()); } - auto pa = item.Pairing(); + auto const& pa = item.Pairing(); if (pa) { @@ -104,42 +185,7 @@ void PrintDeviceList(DeviceClass dc) } ShowMessage("Properties:"); - auto p = item.Properties(); - if (p) - { - for (auto y : p) - { - auto s = y.Value().try_as(); - if (s) - { - ShowMessage(L"%s: %s", y.Key().c_str(), s.value().c_str()); - continue; - } - auto i = y.Value().try_as(); - if (i) - { - ShowMessage(L"%s: %d", y.Key().c_str(), i.value()); - continue; - } - auto b = y.Value().try_as(); - if (b) - { - ShowMessage(L"%s: %d", y.Key().c_str(), (int)b.value()); - continue; - } - auto g = y.Value().try_as(); - if (g) - { - wchar_t buffer[39]; - memset(buffer, 0, sizeof(buffer)); - StringFromGUID2(g.value(), buffer, sizeof(buffer) / sizeof(wchar_t)); - buffer[38] = L'\0'; - ShowMessage(L"%s: %s", y.Key().c_str(), buffer); - continue; - } - ShowMessage(L"%s: %s", y.Key().c_str(), L"[?]"); - } - } + PrintProperties(item.Properties()); } } @@ -160,16 +206,16 @@ winrt::hstring GetBuiltInId(DeviceClass dc) { winrt::hstring const key = L"System.Devices.ContainerId"; - auto list = DeviceInformation::FindAllAsync(dc).get(); - for (auto item : list) + auto const& list = DeviceInformation::FindAllAsync(dc).get(); + for (auto const& item : list) { - auto p = item.Properties(); + auto const& p = item.Properties(); if (!p) { continue; } if (!p.HasKey(key)) { continue; } - auto v = p.Lookup(key); - auto g = v.try_as(); + auto const& v = p.Lookup(key); + auto const& g = v.try_as(); if (!g) { continue; } - auto d = g.value(); + auto const& d = g.value(); if ((d.Data1 != 0) || (d.Data2 != 0) || (d.Data3 != 0) || ((*(uint64_t*)d.Data4) != 0xFFFFFFFFFFFFFFFFULL)) { continue; } return item.Id(); } @@ -192,8 +238,8 @@ winrt::hstring GetBuiltInAudioCaptureId() // OK void GetDeviceIds(DeviceClass dc, std::vector& ids) { - auto list = DeviceInformation::FindAllAsync(dc).get(); - for (auto item : list) + auto const& list = DeviceInformation::FindAllAsync(dc).get(); + for (auto const& item : list) { ids.push_back(item.Id()); } @@ -202,8 +248,8 @@ void GetDeviceIds(DeviceClass dc, std::vector& ids) // OK void GetDeviceIdsAndNames(DeviceClass dc, std::vector& ids, std::vector& names) { - auto list = DeviceInformation::FindAllAsync(dc).get(); - for (auto item : list) + auto const& list = DeviceInformation::FindAllAsync(dc).get(); + for (auto const& item : list) { ids.push_back(item.Id()); names.push_back(item.Name()); diff --git a/hl2ss/hl2ss/nfo.h b/hl2ss/hl2ss/nfo.h index 4e0fa9c86..de5d8ec91 100644 --- a/hl2ss/hl2ss/nfo.h +++ b/hl2ss/hl2ss/nfo.h @@ -5,10 +5,14 @@ #include #include +#include void GetApplicationVersion(uint16_t data[4]); void GetLocalIPv4Address(std::vector &address); void PrintSystemInfo(); +void PrintProperties(winrt::Windows::Foundation::Collections::IMapView const& p); +void PrintProperties(winrt::Windows::Foundation::Collections::IMapView const& p); +void PrintProperties(winrt::Windows::Foundation::Collections::IPropertySet const& p); winrt::hstring GetBuiltInVideoCaptureId(); winrt::hstring GetBuiltInAudioCaptureId(); void GetVideoCaptureIds(std::vector& ids); diff --git a/hl2ss/hl2ss/personal_video.cpp b/hl2ss/hl2ss/personal_video.cpp index 4dc200f9d..9007fe7a8 100644 --- a/hl2ss/hl2ss/personal_video.cpp +++ b/hl2ss/hl2ss/personal_video.cpp @@ -48,7 +48,7 @@ static void PersonalVideo_OnFailed(MediaCapture const&, MediaCaptureFailedEventA // OK static bool PersonalVideo_FindMediaSourceGroup(uint32_t width, uint32_t height, double framerate, MediaFrameSourceGroup &sourceGroup, MediaCaptureVideoProfile &profile, MediaCaptureVideoProfileMediaDescription &description) { - auto mediaFrameSourceGroup = MediaFrameSourceGroup::FromIdAsync(GetBuiltInVideoCaptureId()).get(); + auto const& mediaFrameSourceGroup = MediaFrameSourceGroup::FromIdAsync(GetBuiltInVideoCaptureId()).get(); for (auto const& knownVideoProfile : MediaCapture::FindKnownVideoProfiles(mediaFrameSourceGroup.Id(), KnownVideoProfile::VideoConferencing)) { @@ -378,3 +378,68 @@ void PersonalVideo_SetBacklightCompensation(bool enable) if (!g_ready || g_shared) { return; } g_mediaCapture.VideoDeviceController().BacklightCompensation().TrySetValue(enable ? 1.0 : 0.0); } + +// OK +void PersonalVideo_SetDesiredOptimization(uint32_t mode) +{ + CriticalSection cs(&g_lock); + if (!g_ready || g_shared) { return; } + if (mode > 6) { return; } + g_mediaCapture.VideoDeviceController().DesiredOptimization((MediaCaptureOptimization)mode); +} + +// OK +void PersonalVideo_SetPrimaryUse(uint32_t mode) +{ + CriticalSection cs(&g_lock); + if (!g_ready || g_shared) { return; } + if (mode > 2) { return; } + g_mediaCapture.VideoDeviceController().PrimaryUse((CaptureUse)mode); +} + +// OK +void PersonalVideo_SetOpticalImageStabilization(uint32_t mode) +{ + CriticalSection cs(&g_lock); + if (!g_ready || g_shared) { return; } + if (mode > 1) { return; } + g_mediaCapture.VideoDeviceController().OpticalImageStabilizationControl().Mode((OpticalImageStabilizationMode)mode); +} + +// OK +void PersonalVideo_SetHdrVideo(uint32_t mode) +{ + CriticalSection cs(&g_lock); + if (!g_ready || g_shared) { return; } + if (mode > 2) { return; } + g_mediaCapture.VideoDeviceController().HdrVideoControl().Mode((HdrVideoMode)mode); +} + +// OK +void PersonalVideo_SetRegionsOfInterest(bool clear, bool set, bool auto_exposure, bool auto_focus, bool bounds_normalized, float x, float y, float w, float h, uint32_t type, uint32_t weight) +{ + RegionOfInterest roi; + + CriticalSection cs(&g_lock); + if (!g_ready || g_shared) { return; } + + auto const& mc = g_mediaCapture.VideoDeviceController().RegionsOfInterestControl(); + + if (clear) + { + mc.ClearRegionsAsync().get(); + } + + if (set) + { + roi.AutoExposureEnabled(auto_exposure); + roi.AutoFocusEnabled(auto_focus); + roi.AutoWhiteBalanceEnabled(false); + roi.Bounds({ x, y, w, h }); + roi.BoundsNormalized(bounds_normalized); + roi.Type((RegionOfInterestType)(type & 1)); + roi.Weight(weight % 101); + + mc.SetRegionsAsync({ roi }).get(); + } +} diff --git a/hl2ss/hl2ss/personal_video.h b/hl2ss/hl2ss/personal_video.h index a67a34ad0..0f094751e 100644 --- a/hl2ss/hl2ss/personal_video.h +++ b/hl2ss/hl2ss/personal_video.h @@ -27,3 +27,8 @@ void PersonalVideo_SetExposurePriorityVideo(uint32_t enabled); void PersonalVideo_SetSceneMode(uint32_t mode); void PersonalVideo_SetIsoSpeed(uint32_t setauto, uint32_t value); void PersonalVideo_SetBacklightCompensation(bool enable); +void PersonalVideo_SetDesiredOptimization(uint32_t mode); +void PersonalVideo_SetPrimaryUse(uint32_t mode); +void PersonalVideo_SetOpticalImageStabilization(uint32_t mode); +void PersonalVideo_SetHdrVideo(uint32_t mode); +void PersonalVideo_SetRegionsOfInterest(bool clear, bool set, bool auto_exposure, bool auto_focus, bool bounds_normalized, float x, float y, float w, float h, uint32_t type, uint32_t weight); diff --git a/hl2ss/hl2ss/ports.h b/hl2ss/hl2ss/ports.h index f0123f2b1..db0c4ae20 100644 --- a/hl2ss/hl2ss/ports.h +++ b/hl2ss/hl2ss/ports.h @@ -1,6 +1,7 @@ #pragma once +#define PORT_NUMBER_BASE 3800 #define PORT_NUMBER_RM_VLF 3800 #define PORT_NUMBER_RM_VLL 3801 #define PORT_NUMBER_RM_VRF 3802 diff --git a/hl2ss/hl2ss/research_mode.cpp b/hl2ss/hl2ss/research_mode.cpp index 9f81ecc71..129b133a1 100644 --- a/hl2ss/hl2ss/research_mode.cpp +++ b/hl2ss/hl2ss/research_mode.cpp @@ -159,7 +159,7 @@ bool ResearchMode_Initialize() // OK void ResearchMode_Cleanup() { - for (auto sensor : g_sensors) { if (sensor) { sensor->Release(); } } + for (auto const& sensor : g_sensors) { if (sensor) { sensor->Release(); } } if (g_camera_consent_event) { CloseHandle(g_camera_consent_event); } if (g_imu_consent_event) { CloseHandle(g_imu_consent_event); } @@ -336,3 +336,9 @@ SpatialLocator ResearchMode_GetLocator() { return g_locator; } + +// OK +void ResearchMode_SetEyeSelection(bool enable) +{ + if (enable) { g_pSensorDevice->EnableEyeSelection(); } else { g_pSensorDevice->DisableEyeSelection(); } +} diff --git a/hl2ss/hl2ss/research_mode.h b/hl2ss/hl2ss/research_mode.h index 6959112bc..23ce93986 100644 --- a/hl2ss/hl2ss/research_mode.h +++ b/hl2ss/hl2ss/research_mode.h @@ -43,3 +43,5 @@ bool ResearchMode_GetIntrinsics(IResearchModeSensor* sensor, std::vector& bool ResearchMode_GetExtrinsics(IResearchModeSensor* sensor, DirectX::XMFLOAT4X4& extrinsics); winrt::Windows::Perception::Spatial::SpatialLocator ResearchMode_GetLocator(); + +void ResearchMode_SetEyeSelection(bool enable); diff --git a/hl2ss/hl2ss/scene_understanding.cpp b/hl2ss/hl2ss/scene_understanding.cpp index 6b6af171f..682e76ec7 100644 --- a/hl2ss/hl2ss/scene_understanding.cpp +++ b/hl2ss/hl2ss/scene_understanding.cpp @@ -130,7 +130,7 @@ static void SceneUnderstanding_AdvanceScene(std::shared_ptr scene) g_result.selected.push_back(std::dynamic_pointer_cast(component)); } - for (auto& object : scene->GetSceneObjects()) { if ((SceneUnderstanding_KindToFlag(object->GetKind()) & g_query.kinds) && (check.count(object->GetId()) <= 0)) { g_result.selected.push_back(object); } } + for (auto const& object : scene->GetSceneObjects()) { if ((SceneUnderstanding_KindToFlag(object->GetKind()) & g_query.kinds) && (check.count(object->GetId()) <= 0)) { g_result.selected.push_back(object); } } } // OK diff --git a/hl2ss/hl2ss/spatial_input.cpp b/hl2ss/hl2ss/spatial_input.cpp index 2a708281d..4c85dc6bf 100644 --- a/hl2ss/hl2ss/spatial_input.cpp +++ b/hl2ss/hl2ss/spatial_input.cpp @@ -67,7 +67,7 @@ int SpatialInput_GetHeadPoseAndEyeRay(SpatialCoordinateSystem const& world, Perc if (!pointer) { return ret; } - auto h = pointer.Head(); + auto const& h = pointer.Head(); head_pose.position = h.Position(); head_pose.forward = h.ForwardDirection(); @@ -75,13 +75,13 @@ int SpatialInput_GetHeadPoseAndEyeRay(SpatialCoordinateSystem const& world, Perc ret |= 1; - auto pose = pointer.Eyes(); + auto const& pose = pointer.Eyes(); if (!pose) { return ret; } - auto gaze = pose.Gaze(); + auto const& gaze = pose.Gaze(); if (!gaze) { return ret; } - auto spatialRay = gaze.Value(); + auto const& spatialRay = gaze.Value(); eye_ray.origin = spatialRay.Origin; eye_ray.direction = spatialRay.Direction; @@ -94,13 +94,13 @@ int SpatialInput_GetHeadPoseAndEyeRay(SpatialCoordinateSystem const& world, Perc // OK int SpatialInput_GetHandPose(SpatialCoordinateSystem const& world, PerceptionTimestamp const& ts, std::vector& left_poses, std::vector& right_poses) { - auto source_states = g_sim.GetDetectedSourcesAtTimestamp(ts); + auto const& source_states = g_sim.GetDetectedSourcesAtTimestamp(ts); int ret = 0; std::vector* target; int id; bool ok; - for (auto source_state : source_states) + for (auto const& source_state : source_states) { if (source_state.Source().Kind() != SpatialInteractionSourceKind::Hand) { continue; } @@ -111,7 +111,7 @@ int SpatialInput_GetHandPose(SpatialCoordinateSystem const& world, PerceptionTim default: continue; } - auto pose = source_state.TryGetHandPose(); + auto const& pose = source_state.TryGetHandPose(); if (!pose) { continue; } ok = pose.TryGetJoints(world, g_joints, *target); diff --git a/hl2ss/hl2ss/spatial_mapping.cpp b/hl2ss/hl2ss/spatial_mapping.cpp index 767a745d9..15a1a9592 100644 --- a/hl2ss/hl2ss/spatial_mapping.cpp +++ b/hl2ss/hl2ss/spatial_mapping.cpp @@ -105,7 +105,7 @@ void SpatialMapping_GetObservedSurfaces(SpatialMapping_SurfaceInfo const*& data, g_observed = g_sso.GetObservedSurfaces(); int i = 0; g_observed_ids.resize(g_observed.Size()); - for (auto pair : g_observed) { g_observed_ids[i++] = { pair.Key(), pair.Value().UpdateTime().time_since_epoch().count() }; } + for (auto const& pair : g_observed) { g_observed_ids[i++] = { pair.Key(), pair.Value().UpdateTime().time_since_epoch().count() }; } data = g_observed_ids.data(); size = g_observed_ids.size(); } @@ -113,7 +113,7 @@ void SpatialMapping_GetObservedSurfaces(SpatialMapping_SurfaceInfo const*& data, // OK static SpatialBoundingOrientedBox SpatialMapping_GetBounds(SpatialSurfaceInfo ssi, SpatialSurfaceMesh ssm) { - auto sbob = ssi.TryGetBounds(ssm.CoordinateSystem()); + auto const& sbob = ssi.TryGetBounds(ssm.CoordinateSystem()); return sbob ? sbob.Value() : SpatialBoundingOrientedBox{ {0, 0, 0}, {0, 0, 0}, {0, 0, 0, 0} }; } diff --git a/hl2ss/hl2ss/stream_ea.cpp b/hl2ss/hl2ss/stream_ea.cpp index 1ba74eeb3..1df1c21c2 100644 --- a/hl2ss/hl2ss/stream_ea.cpp +++ b/hl2ss/hl2ss/stream_ea.cpp @@ -3,6 +3,7 @@ #include "custom_media_sink.h" #include "custom_media_buffers.h" #include "extended_audio.h" +#include "extended_execution.h" #include "neon.h" #include "log.h" #include "ports.h" @@ -29,8 +30,8 @@ using namespace winrt::Windows::Foundation::Numerics; static HANDLE g_thread = NULL; // CloseHandle static HANDLE g_event_quit = NULL; // CloseHandle +static SRWLOCK g_lock; -static bool g_reader_status = false; static IMFSinkWriter* g_pSinkWriter = nullptr; // Release static DWORD g_dwAudioIndex = 0; @@ -42,12 +43,11 @@ static DWORD g_dwAudioIndex = 0; static void EA_DeviceQuery(SOCKET clientsocket) { winrt::hstring query; - uint32_t bytes; WSABUF wsaBuf[2]; ExtendedAudio_QueryDevices(query); - bytes = query.size() * sizeof(wchar_t); + uint32_t bytes = query.size() * sizeof(wchar_t); pack_buffer(wsaBuf, 0, &bytes, sizeof(bytes)); pack_buffer(wsaBuf, 1, query.c_str(), bytes); @@ -60,24 +60,22 @@ static void EA_OnAudioFrameArrived(MediaFrameReader const& sender, MediaFrameArr { (void)args; - MediaFrameReference frame = nullptr; IMFSample* pSample; // Release IMFMediaBuffer* pBuffer; // Release BYTE* pDst; - int64_t timestamp; - if (!g_reader_status) { return; } - frame = sender.TryAcquireLatestFrame(); - if (!frame) { return; } - - timestamp = frame.SystemRelativeTime().Value().count(); + if (TryAcquireSRWLockShared(&g_lock) != 0) + { + auto const& frame = sender.TryAcquireLatestFrame(); + if (frame) + { + int64_t timestamp = frame.SystemRelativeTime().Value().count(); AudioMediaFrame amf = frame.AudioMediaFrame(); AudioFrame audio = amf.GetAudioFrame(); AudioBuffer buffer = audio.LockBuffer(AudioBufferAccessMode::Read); - auto reference = buffer.CreateReference(); - - auto aep = amf.AudioEncodingProperties(); + auto const& reference = buffer.CreateReference(); + auto const& aep = amf.AudioEncodingProperties(); uint32_t bytes_per_sample = aep.BitsPerSample() / 8; uint32_t channels = aep.ChannelCount(); @@ -122,6 +120,9 @@ static void EA_OnAudioFrameArrived(MediaFrameReader const& sender, MediaFrameArr pBuffer->Release(); pSample->Release(); + } + ReleaseSRWLockShared(&g_lock); + } } // OK @@ -132,10 +133,8 @@ static void EA_SendSample(IMFSample* pSample, void* param) BYTE* pBytes; DWORD cbData; WSABUF wsaBuf[3]; - HookCallbackSocket* user; - bool ok; - user = (HookCallbackSocket*)param; + HookCallbackSocket* user = (HookCallbackSocket*)param; pSample->GetSampleTime(&sampletime); pSample->ConvertToContiguousBuffer(&pBuffer); @@ -146,7 +145,7 @@ static void EA_SendSample(IMFSample* pSample, void* param) pack_buffer(wsaBuf, 1, &cbData, sizeof(cbData)); pack_buffer(wsaBuf, 2, pBytes, cbData); - ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); + bool ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); if (!ok) { SetEvent(user->clientevent); } pBuffer->Unlock(); @@ -159,7 +158,6 @@ static void EA_Shoutcast(SOCKET clientsocket) uint32_t const channels = 2; uint32_t const samplerate = 48000; - MediaFrameReader reader = nullptr; CustomMediaSink* pSink; // Release HANDLE event_client; // CloseHandle MRCAudioOptions options; @@ -188,6 +186,8 @@ static void EA_Shoutcast(SOCKET clientsocket) event_client = CreateEvent(NULL, TRUE, FALSE, NULL); ExtendedAudio_RegisterEvent(event_client); + auto const& reader = ExtendedAudio_CreateFrameReader(); + reader.AcquisitionMode(MediaFrameReaderAcquisitionMode::Buffered); user.clientsocket = clientsocket; user.clientevent = event_client; @@ -195,22 +195,20 @@ static void EA_Shoutcast(SOCKET clientsocket) CreateSinkWriterAudio(&pSink, &g_pSinkWriter, &g_dwAudioIndex, AudioSubtype::AudioSubtype_S16, format, EA_SendSample, &user); - reader = ExtendedAudio_CreateFrameReader(); - - reader.AcquisitionMode(MediaFrameReaderAcquisitionMode::Buffered); reader.FrameArrived(EA_OnAudioFrameArrived); - g_reader_status = true; + ReleaseSRWLockExclusive(&g_lock); reader.StartAsync().get(); WaitForSingleObject(event_client, INFINITE); - g_reader_status = false; reader.StopAsync().get(); + AcquireSRWLockExclusive(&g_lock); g_pSinkWriter->Flush(g_dwAudioIndex); g_pSinkWriter->Release(); pSink->Shutdown(); pSink->Release(); + reader.Close(); ExtendedAudio_RegisterEvent(NULL); CloseHandle(event_client); @@ -223,11 +221,16 @@ static DWORD WINAPI EA_EntryPoint(void*) { SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket + int base_priority; listensocket = CreateSocket(PORT_NAME_EA); ShowMessage("EA: Listening at port %s", PORT_NAME_EA); + AcquireSRWLockExclusive(&g_lock); + + base_priority = GetThreadPriority(GetCurrentThread()); + do { ShowMessage("EA: Waiting for client"); @@ -237,8 +240,12 @@ static DWORD WINAPI EA_EntryPoint(void*) ShowMessage("EA: Client connected"); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(PORT_NUMBER_EA - PORT_NUMBER_BASE)); + EA_Shoutcast(clientsocket); + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("EA: Client disconnected"); @@ -255,6 +262,7 @@ static DWORD WINAPI EA_EntryPoint(void*) // OK void EA_Initialize() { + InitializeSRWLock(&g_lock); g_event_quit = CreateEvent(NULL, TRUE, FALSE, NULL); g_thread = CreateThread(NULL, 0, EA_EntryPoint, NULL, 0, NULL); } diff --git a/hl2ss/hl2ss/stream_eet.cpp b/hl2ss/hl2ss/stream_eet.cpp index 5b7e63b3b..a2e6010e7 100644 --- a/hl2ss/hl2ss/stream_eet.cpp +++ b/hl2ss/hl2ss/stream_eet.cpp @@ -2,6 +2,7 @@ #include "server.h" #include "locator.h" #include "extended_eye_tracking.h" +#include "extended_execution.h" #include "ports.h" #include "timestamps.h" #include "log.h" @@ -36,7 +37,7 @@ struct EET_Packet { uint64_t timestamp; uint32_t size; - uint32_t reserved; + uint32_t _reserved; EET_Frame frame; float4x4 pose; }; @@ -92,7 +93,7 @@ static void EET_Stream(SOCKET clientsocket, SpatialLocator const &locator, uint6 pack_buffer(wsaBuf, 0, &eet_packet, sizeof(eet_packet)); eet_packet.size = sizeof(uint32_t) + sizeof(EET_Frame); - eet_packet.reserved = 0; + eet_packet._reserved = 0; do { @@ -139,6 +140,7 @@ static DWORD WINAPI EET_EntryPoint(void* param) uint64_t utc_offset; SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket + int base_priority; ShowMessage("EET: Waiting for consent"); @@ -150,6 +152,8 @@ static DWORD WINAPI EET_EntryPoint(void* param) ShowMessage("EET: Listening at port %s", PORT_NAME_EET); + base_priority = GetThreadPriority(GetCurrentThread()); + do { ShowMessage("EET: Waiting for client"); @@ -159,8 +163,12 @@ static DWORD WINAPI EET_EntryPoint(void* param) ShowMessage("EET: Client connected"); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(PORT_NUMBER_EET - PORT_NUMBER_BASE)); + EET_Stream(clientsocket, locator, utc_offset); + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("EET: Client disconnected"); diff --git a/hl2ss/hl2ss/stream_ev.cpp b/hl2ss/hl2ss/stream_ev.cpp index 79b068fca..d2cc830f2 100644 --- a/hl2ss/hl2ss/stream_ev.cpp +++ b/hl2ss/hl2ss/stream_ev.cpp @@ -7,6 +7,7 @@ #include "ports.h" #include "timestamps.h" #include "ipc_sc.h" +#include "extended_execution.h" #include #include @@ -19,10 +20,25 @@ using namespace winrt::Windows::Media::Capture::Frames; using namespace winrt::Windows::Media::Devices::Core; using namespace winrt::Windows::Foundation::Numerics; +struct uint64x2 +{ + uint64_t val[2]; +}; + struct PV_Projection { - float2 f; - float2 c; + float2 f; + float2 c; + uint64_t exposure_time; + uint64x2 exposure_compensation; + uint32_t lens_position; + uint32_t focus_state; + uint32_t iso_speed; + uint32_t white_balance; + float2 iso_gains; + float3 white_balance_gains; + uint32_t _reserved; + uint64_t timestamp; float4x4 pose; }; @@ -32,13 +48,14 @@ struct PV_Projection static HANDLE g_event_quit = NULL; // CloseHandle static HANDLE g_thread = NULL; // CloseHandle -static bool g_reader_status = false; +static SRWLOCK g_lock; // Mode: 0, 1 static IMFSinkWriter* g_pSinkWriter = NULL; // Release static DWORD g_dwVideoIndex = 0; static uint32_t g_counter = 0; static uint32_t g_divisor = 1; +static PV_Projection g_pvp_sh; //----------------------------------------------------------------------------- // Functions @@ -48,36 +65,43 @@ static uint32_t g_divisor = 1; void EV_OnVideoFrameArrived(MediaFrameReader const& sender, MediaFrameArrivedEventArgs const& args) { (void)args; - - CameraIntrinsics intrinsics = nullptr; - MediaFrameReference frame = nullptr; + IMFSample* pSample; // Release SoftwareBitmapBuffer* pBuffer; // Release - int64_t timestamp; - - if (!g_reader_status) { return; } - frame = sender.TryAcquireLatestFrame(); - if (!frame) { return; } + PV_Projection pj; + if (TryAcquireSRWLockShared(&g_lock) != 0) + { + auto const& frame = sender.TryAcquireLatestFrame(); + if (frame) + { if (g_counter == 0) { SoftwareBitmapBuffer::CreateInstance(&pBuffer, frame); MFCreateSample(&pSample); - timestamp = frame.SystemRelativeTime().Value().count(); + int64_t timestamp = frame.SystemRelativeTime().Value().count(); pSample->AddBuffer(pBuffer); pSample->SetSampleDuration(frame.Duration().count()); pSample->SetSampleTime(timestamp); + memset(&pj, 0, sizeof(pj)); + + pj.timestamp = timestamp; + + pSample->SetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&pj, sizeof(pj)); + g_pSinkWriter->WriteSample(g_dwVideoIndex, pSample); pSample->Release(); pBuffer->Release(); } - g_counter = (g_counter + 1) % g_divisor; + } + ReleaseSRWLockShared(&g_lock); + } } // OK @@ -88,38 +112,39 @@ void EV_SendSample(IMFSample* pSample, void* param) LONGLONG sampletime; BYTE* pBytes; DWORD cbData; - DWORD cbDataEx; - PV_Projection pj; - WSABUF wsaBuf[ENABLE_LOCATION ? 6 : 5]; - HookCallbackSocket* user; - bool ok; + WSABUF wsaBuf[ENABLE_LOCATION ? 5 : 4]; - user = (HookCallbackSocket*)param; + HookCallbackSocket* user = (HookCallbackSocket*)param; + H26xFormat* format = (H26xFormat*)user->format; + bool sh = format->profile != H26xProfile::H26xProfile_None; pSample->GetSampleTime(&sampletime); pSample->ConvertToContiguousBuffer(&pBuffer); - memset(&pj, 0, sizeof(pj)); + + if (!sh) { pSample->GetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&g_pvp_sh, sizeof(g_pvp_sh), NULL); } pBuffer->Lock(&pBytes, NULL, &cbData); - cbDataEx = cbData + sizeof(pj.f) + sizeof(pj.c); + int const metadata = sizeof(g_pvp_sh) - sizeof(g_pvp_sh.timestamp) - sizeof(g_pvp_sh.pose); + DWORD cbDataEx = cbData + metadata; - pack_buffer(wsaBuf, 0, &sampletime, sizeof(sampletime)); + pack_buffer(wsaBuf, 0, &g_pvp_sh.timestamp, sizeof(g_pvp_sh.timestamp)); pack_buffer(wsaBuf, 1, &cbDataEx, sizeof(cbDataEx)); pack_buffer(wsaBuf, 2, pBytes, cbData); - pack_buffer(wsaBuf, 3, &pj.f, sizeof(pj.f)); - pack_buffer(wsaBuf, 4, &pj.c, sizeof(pj.c)); + pack_buffer(wsaBuf, 3, &g_pvp_sh, metadata); if constexpr (ENABLE_LOCATION) { - pack_buffer(wsaBuf, 5, &pj.pose, sizeof(pj.pose)); + pack_buffer(wsaBuf, 4, &g_pvp_sh.pose, sizeof(g_pvp_sh.pose)); } - ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); + bool ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); if (!ok) { SetEvent(user->clientevent); } pBuffer->Unlock(); pBuffer->Release(); + + if (sh) { pSample->GetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&g_pvp_sh, sizeof(g_pvp_sh), NULL); } } // OK @@ -150,12 +175,13 @@ void EV_Stream(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& g_counter = 0; g_divisor = format.divisor; + memset(&g_pvp_sh, 0, sizeof(g_pvp_sh)); - g_reader_status = true; + ReleaseSRWLockExclusive(&g_lock); reader.StartAsync().get(); WaitForSingleObject(clientevent, INFINITE); - g_reader_status = false; reader.StopAsync().get(); + AcquireSRWLockExclusive(&g_lock); g_pSinkWriter->Flush(g_dwVideoIndex); g_pSinkWriter->Release(); @@ -167,12 +193,11 @@ void EV_Stream(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& static void EV_DeviceQuery(SOCKET clientsocket) { winrt::hstring query; - uint32_t bytes; WSABUF wsaBuf[2]; ExtendedVideo_QueryDevices(query); - bytes = query.size() * sizeof(wchar_t); + uint32_t bytes = query.size() * sizeof(wchar_t); pack_buffer(wsaBuf, 0, &bytes, sizeof(bytes)); pack_buffer(wsaBuf, 1, query.c_str(), bytes); @@ -183,7 +208,6 @@ static void EV_DeviceQuery(SOCKET clientsocket) // OK static void EV_Stream(SOCKET clientsocket, uint8_t mode, H26xFormat& format) { - MediaFrameReader videoFrameReader = nullptr; HANDLE clientevent; // CloseHandle VideoSubtype subtype; bool ok; @@ -194,7 +218,7 @@ static void EV_Stream(SOCKET clientsocket, uint8_t mode, H26xFormat& format) clientevent = CreateEvent(NULL, TRUE, FALSE, NULL); ExtendedVideo_RegisterEvent(clientevent); - videoFrameReader = ExtendedVideo_CreateFrameReader(); + auto const& videoFrameReader = ExtendedVideo_CreateFrameReader(); videoFrameReader.AcquisitionMode(MediaFrameReaderAcquisitionMode::Buffered); switch (mode & 3) @@ -212,6 +236,7 @@ static void EV_Stream(SOCKET clientsocket, uint8_t mode, H26xFormat& format) // OK static void EV_Stream(SOCKET clientsocket) { + MRCVideoOptions options; H26xFormat format; uint8_t mode; bool ok; @@ -230,7 +255,6 @@ static void EV_Stream(SOCKET clientsocket) if (mode & 4) { - MRCVideoOptions options; ok = ReceiveMRCVideoOptions(clientsocket, options); if (!ok) { return; } if (ExtendedVideo_Status()) { ExtendedVideo_Close(); } @@ -251,11 +275,16 @@ static DWORD WINAPI EV_EntryPoint(void *param) SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket + int base_priority; listensocket = CreateSocket(PORT_NAME_EV); ShowMessage("EV: Listening at port %s", PORT_NAME_EV); + AcquireSRWLockExclusive(&g_lock); + + base_priority = GetThreadPriority(GetCurrentThread()); + do { ShowMessage("EV: Waiting for client"); @@ -265,8 +294,12 @@ static DWORD WINAPI EV_EntryPoint(void *param) ShowMessage("EV: Client connected"); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(PORT_NUMBER_EV - PORT_NUMBER_BASE)); + EV_Stream(clientsocket); + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("EV: Client disconnected"); @@ -283,6 +316,7 @@ static DWORD WINAPI EV_EntryPoint(void *param) // OK void EV_Initialize() { + InitializeSRWLock(&g_lock); g_event_quit = CreateEvent(NULL, TRUE, FALSE, NULL); g_thread = CreateThread(NULL, 0, EV_EntryPoint, NULL, 0, NULL); } diff --git a/hl2ss/hl2ss/stream_mc.cpp b/hl2ss/hl2ss/stream_mc.cpp index 70acf0ad9..601d5fcb0 100644 --- a/hl2ss/hl2ss/stream_mc.cpp +++ b/hl2ss/hl2ss/stream_mc.cpp @@ -4,6 +4,7 @@ #include "ports.h" #include "microphone_capture.h" #include "ipc_sc.h" +#include "extended_execution.h" #include "log.h" #include @@ -114,11 +115,14 @@ static DWORD WINAPI MC_EntryPoint(void*) { SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket + int base_priority; listensocket = CreateSocket(PORT_NAME_MC); ShowMessage("MC: Listening at port %s", PORT_NAME_MC); + base_priority = GetThreadPriority(GetCurrentThread()); + do { ShowMessage("MC: Waiting for client"); @@ -128,8 +132,12 @@ static DWORD WINAPI MC_EntryPoint(void*) ShowMessage("MC: Client connected"); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(PORT_NUMBER_MC - PORT_NUMBER_BASE)); + MC_Shoutcast(clientsocket); + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("MC: Client disconnected"); diff --git a/hl2ss/hl2ss/stream_pv.cpp b/hl2ss/hl2ss/stream_pv.cpp index 765a5eeae..c32d04a5f 100644 --- a/hl2ss/hl2ss/stream_pv.cpp +++ b/hl2ss/hl2ss/stream_pv.cpp @@ -9,6 +9,8 @@ #include "timestamps.h" #include "ipc_sc.h" #include "research_mode.h" +#include "extended_execution.h" +#include "nfo.h" #include #include @@ -21,22 +23,55 @@ using namespace winrt::Windows::Media::Capture; using namespace winrt::Windows::Media::Capture::Frames; using namespace winrt::Windows::Media::Devices::Core; using namespace winrt::Windows::Foundation::Numerics; +using namespace winrt::Windows::Foundation::Collections; using namespace winrt::Windows::Perception::Spatial; +struct uint64x2 +{ + uint64_t val[2]; +}; + +struct float7 +{ + float val[7]; +}; + struct PV_Projection { - float2 f; - float2 c; + float2 f; + float2 c; + uint64_t exposure_time; + uint64x2 exposure_compensation; + uint32_t lens_position; + uint32_t focus_state; + uint32_t iso_speed; + uint32_t white_balance; + float2 iso_gains; + float3 white_balance_gains; + uint32_t _reserved; + uint64_t timestamp; float4x4 pose; }; +struct PV_Mode2 +{ + float2 f; + float2 c; + float3 r; + float2 t; + float4x4 p; + float4x4 extrinsics; + float4 intrinsics_mf; + float7 extrinsics_mf; +}; + //----------------------------------------------------------------------------- // Global Variables //----------------------------------------------------------------------------- static HANDLE g_event_quit = NULL; // CloseHandle static HANDLE g_thread = NULL; // CloseHandle -static bool g_reader_status = false; +static SRWLOCK g_lock; // Mode: 0, 1 static IMFSinkWriter* g_pSinkWriter = NULL; // Release @@ -47,8 +82,7 @@ static PV_Projection g_pvp_sh; // Mode: 2 static HANDLE g_event_intrinsic = NULL; // alias -static float g_intrinsics[2 + 2 + 3 + 2 + 16]; -static float4x4 g_extrinsics; +static PV_Mode2 g_calibration; //----------------------------------------------------------------------------- // Functions @@ -60,34 +94,44 @@ void PV_OnVideoFrameArrived(MediaFrameReader const& sender, MediaFrameArrivedEve { (void)args; - CameraIntrinsics intrinsics = nullptr; - MediaFrameReference frame = nullptr; IMFSample* pSample; // Release SoftwareBitmapBuffer* pBuffer; // Release PV_Projection pj; - int64_t timestamp; - - if (!g_reader_status) { return; } - frame = sender.TryAcquireLatestFrame(); - if (!frame) { return; } + if (TryAcquireSRWLockShared(&g_lock) != 0) + { + auto const& frame = sender.TryAcquireLatestFrame(); + if (frame) + { if (g_counter == 0) { SoftwareBitmapBuffer::CreateInstance(&pBuffer, frame); MFCreateSample(&pSample); - timestamp = frame.SystemRelativeTime().Value().count(); + int64_t timestamp = frame.SystemRelativeTime().Value().count(); pSample->AddBuffer(pBuffer); pSample->SetSampleDuration(frame.Duration().count()); pSample->SetSampleTime(timestamp); - intrinsics = frame.VideoMediaFrame().CameraIntrinsics(); + auto const& intrinsics = frame.VideoMediaFrame().CameraIntrinsics(); + auto const& metadata = frame.Properties().Lookup(MFSampleExtension_CaptureMetadata).as>(); + + pj.timestamp = timestamp; pj.f = intrinsics.FocalLength(); pj.c = intrinsics.PrincipalPoint(); + pj.exposure_time = metadata.Lookup(MF_CAPTURE_METADATA_EXPOSURE_TIME).as(); + pj.exposure_compensation = *(uint64x2*)metadata.Lookup(MF_CAPTURE_METADATA_EXPOSURE_COMPENSATION).as>().Value().begin(); + pj.iso_speed = metadata.Lookup(MF_CAPTURE_METADATA_ISO_SPEED).as(); + pj.iso_gains = *(float2*)metadata.Lookup(MF_CAPTURE_METADATA_ISO_GAINS).as>().Value().begin(); + pj.lens_position = metadata.Lookup(MF_CAPTURE_METADATA_LENS_POSITION).as(); + pj.focus_state = metadata.Lookup(MF_CAPTURE_METADATA_FOCUSSTATE).as(); + pj.white_balance = metadata.Lookup(MF_CAPTURE_METADATA_WHITEBALANCE).as(); + pj.white_balance_gains = *(float3*)metadata.Lookup(MF_CAPTURE_METADATA_WHITEBALANCE_GAINS).as>().Value().begin(); + if constexpr (ENABLE_LOCATION) { pj.pose = Locator_GetTransformTo(frame.CoordinateSystem(), Locator_GetWorldCoordinateSystem(QPCTimestampToPerceptionTimestamp(timestamp))); @@ -100,8 +144,10 @@ void PV_OnVideoFrameArrived(MediaFrameReader const& sender, MediaFrameArrivedEve pSample->Release(); pBuffer->Release(); } - g_counter = (g_counter + 1) % g_divisor; + } + ReleaseSRWLockShared(&g_lock); + } } // OK @@ -109,38 +155,33 @@ static void PV_OnVideoFrameArrived_Intrinsics(MediaFrameReader const& sender, Me { (void)args; - CameraIntrinsics intrinsics = nullptr; - MediaFrameReference frame = nullptr; - float2 f; - float2 c; - float3 r; - float2 t; - float4x4 p; - DWORD status; - - frame = sender.TryAcquireLatestFrame(); - if (!frame) { return; } - - status = WaitForSingleObject(g_event_intrinsic, 0); - if (status != WAIT_TIMEOUT) { return; } - - intrinsics = frame.VideoMediaFrame().CameraIntrinsics(); + if (TryAcquireSRWLockExclusive(&g_lock) != 0) + { + if (WaitForSingleObject(g_event_intrinsic, 0) == WAIT_TIMEOUT) + { + auto const& frame = sender.TryAcquireLatestFrame(); + if (frame) + { + auto const& intrinsics = frame.VideoMediaFrame().CameraIntrinsics(); + auto const& extrinsics = frame.Properties().Lookup(MFSampleExtension_CameraExtrinsics).as>().Value(); + auto const& additional = frame.Format().Properties().Lookup(winrt::guid("86b6adbb-3735-447d-bee5-6fc23cb58d4a")).as>().Value(); - f = intrinsics.FocalLength(); - c = intrinsics.PrincipalPoint(); - r = intrinsics.RadialDistortion(); - t = intrinsics.TangentialDistortion(); - p = intrinsics.UndistortedProjectionTransform(); + g_calibration.f = intrinsics.FocalLength(); + g_calibration.c = intrinsics.PrincipalPoint(); + g_calibration.r = intrinsics.RadialDistortion(); + g_calibration.t = intrinsics.TangentialDistortion(); + g_calibration.p = intrinsics.UndistortedProjectionTransform(); - memcpy(&g_intrinsics[0], &f, sizeof(f)); - memcpy(&g_intrinsics[2], &c, sizeof(c)); - memcpy(&g_intrinsics[4], &r, sizeof(r)); - memcpy(&g_intrinsics[7], &t, sizeof(t)); - memcpy(&g_intrinsics[9], &p, sizeof(p)); + g_calibration.extrinsics = Locator_Locate(QPCTimestampToPerceptionTimestamp(frame.SystemRelativeTime().Value().count()), ResearchMode_GetLocator(), frame.CoordinateSystem()); - g_extrinsics = Locator_Locate(QPCTimestampToPerceptionTimestamp(frame.SystemRelativeTime().Value().count()), ResearchMode_GetLocator(), frame.CoordinateSystem()); + g_calibration.intrinsics_mf = *(float4*)&((float*)additional.begin())[3]; + g_calibration.extrinsics_mf = *(float7*)&((float*)extrinsics.begin())[5]; SetEvent(g_event_intrinsic); + } + } + ReleaseSRWLockExclusive(&g_lock); + } } // OK @@ -151,16 +192,11 @@ void PV_SendSample(IMFSample* pSample, void* param) LONGLONG sampletime; BYTE* pBytes; DWORD cbData; - DWORD cbDataEx; - WSABUF wsaBuf[ENABLE_LOCATION ? 6 : 5]; - HookCallbackSocket* user; - H26xFormat* format; - bool sh; - bool ok; + WSABUF wsaBuf[ENABLE_LOCATION ? 5 : 4]; - user = (HookCallbackSocket*)param; - format = (H26xFormat*)user->format; - sh = format->profile != H26xProfile::H26xProfile_None; + HookCallbackSocket* user = (HookCallbackSocket*)param; + H26xFormat* format = (H26xFormat*)user->format; + bool sh = format->profile != H26xProfile::H26xProfile_None; pSample->GetSampleTime(&sampletime); pSample->ConvertToContiguousBuffer(&pBuffer); @@ -169,20 +205,20 @@ void PV_SendSample(IMFSample* pSample, void* param) pBuffer->Lock(&pBytes, NULL, &cbData); - cbDataEx = cbData + sizeof(g_pvp_sh.f) + sizeof(g_pvp_sh.c); + int const metadata = sizeof(g_pvp_sh) - sizeof(g_pvp_sh.timestamp) - sizeof(g_pvp_sh.pose); + DWORD cbDataEx = cbData + metadata; - pack_buffer(wsaBuf, 0, &sampletime, sizeof(sampletime)); + pack_buffer(wsaBuf, 0, &g_pvp_sh.timestamp, sizeof(g_pvp_sh.timestamp)); pack_buffer(wsaBuf, 1, &cbDataEx, sizeof(cbDataEx)); pack_buffer(wsaBuf, 2, pBytes, cbData); - pack_buffer(wsaBuf, 3, &g_pvp_sh.f, sizeof(g_pvp_sh.f)); - pack_buffer(wsaBuf, 4, &g_pvp_sh.c, sizeof(g_pvp_sh.c)); + pack_buffer(wsaBuf, 3, &g_pvp_sh, metadata); if constexpr(ENABLE_LOCATION) { - pack_buffer(wsaBuf, 5, &g_pvp_sh.pose, sizeof(g_pvp_sh.pose)); + pack_buffer(wsaBuf, 4, &g_pvp_sh.pose, sizeof(g_pvp_sh.pose)); } - ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); + bool ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); if (!ok) { SetEvent(user->clientevent); } pBuffer->Unlock(); @@ -218,15 +254,15 @@ void PV_Stream(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& reader.FrameArrived(PV_OnVideoFrameArrived); g_counter = 0; - g_divisor = format.divisor; + g_divisor = format.divisor; memset(&g_pvp_sh, 0, sizeof(g_pvp_sh)); - g_reader_status = true; + ReleaseSRWLockExclusive(&g_lock); reader.StartAsync().get(); WaitForSingleObject(clientevent, INFINITE); - g_reader_status = false; reader.StopAsync().get(); - + AcquireSRWLockExclusive(&g_lock); + g_pSinkWriter->Flush(g_dwVideoIndex); g_pSinkWriter->Release(); pSink->Shutdown(); @@ -236,18 +272,19 @@ void PV_Stream(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& // OK static void PV_Intrinsics(SOCKET clientsocket, HANDLE clientevent, MediaFrameReader const& reader) { - WSABUF wsaBuf[2]; + WSABUF wsaBuf[1]; g_event_intrinsic = clientevent; reader.FrameArrived(PV_OnVideoFrameArrived_Intrinsics); + ReleaseSRWLockExclusive(&g_lock); reader.StartAsync().get(); - WaitForSingleObject(g_event_intrinsic, INFINITE); + WaitForSingleObject(clientevent, INFINITE); reader.StopAsync().get(); + AcquireSRWLockExclusive(&g_lock); - pack_buffer(wsaBuf, 0, g_intrinsics, sizeof(g_intrinsics)); - pack_buffer(wsaBuf, 1, &g_extrinsics, sizeof(g_extrinsics)); + pack_buffer(wsaBuf, 0, &g_calibration, sizeof(g_calibration)); send_multiple(clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); } @@ -255,8 +292,8 @@ static void PV_Intrinsics(SOCKET clientsocket, HANDLE clientevent, MediaFrameRea // OK static void PV_Stream(SOCKET clientsocket) { - MediaFrameReader videoFrameReader = nullptr; HANDLE clientevent; // CloseHandle + MRCVideoOptions options; H26xFormat format; uint8_t mode; bool ok; @@ -269,7 +306,6 @@ static void PV_Stream(SOCKET clientsocket) if (mode & 4) { - MRCVideoOptions options; ok = ReceiveMRCVideoOptions(clientsocket, options); if (!ok) { return; } if (PersonalVideo_Status()) { PersonalVideo_Close(); } @@ -284,9 +320,9 @@ static void PV_Stream(SOCKET clientsocket) clientevent = CreateEvent(NULL, TRUE, FALSE, NULL); PersonalVideo_RegisterEvent(clientevent); - videoFrameReader = PersonalVideo_CreateFrameReader(); + auto const& videoFrameReader = PersonalVideo_CreateFrameReader(); videoFrameReader.AcquisitionMode(MediaFrameReaderAcquisitionMode::Buffered); - + switch (mode & 3) { case 0: PV_Stream(clientsocket, clientevent, videoFrameReader, format); break; @@ -309,11 +345,16 @@ static DWORD WINAPI PV_EntryPoint(void *param) SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket + int base_priority; listensocket = CreateSocket(PORT_NAME_PV); ShowMessage("PV: Listening at port %s", PORT_NAME_PV); + AcquireSRWLockExclusive(&g_lock); + + base_priority = GetThreadPriority(GetCurrentThread()); + do { ShowMessage("PV: Waiting for client"); @@ -323,8 +364,12 @@ static DWORD WINAPI PV_EntryPoint(void *param) ShowMessage("PV: Client connected"); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(PORT_NUMBER_PV - PORT_NUMBER_BASE)); + PV_Stream(clientsocket); + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("PV: Client disconnected"); @@ -341,6 +386,7 @@ static DWORD WINAPI PV_EntryPoint(void *param) // OK void PV_Initialize() { + InitializeSRWLock(&g_lock); g_event_quit = CreateEvent(NULL, TRUE, FALSE, NULL); g_thread = CreateThread(NULL, 0, PV_EntryPoint, NULL, 0, NULL); } diff --git a/hl2ss/hl2ss/stream_rm.cpp b/hl2ss/hl2ss/stream_rm.cpp index c5114e4df..17921b541 100644 --- a/hl2ss/hl2ss/stream_rm.cpp +++ b/hl2ss/hl2ss/stream_rm.cpp @@ -1,6 +1,7 @@ #include "research_mode.h" #include "server.h" +#include "extended_execution.h" #include "stream_rm_vlc.h" #include "stream_rm_imu.h" #include "stream_rm_zht.h" @@ -69,6 +70,7 @@ static DWORD WINAPI RM_EntryPoint(void* param) SOCKET clientsocket; // closesocket IResearchModeSensor* sensor; ResearchModeSensorType type; + int base_priority; char const* port; bool ok; @@ -85,6 +87,8 @@ static DWORD WINAPI RM_EntryPoint(void* param) listensocket = CreateSocket(port); ShowMessage("RM%d: Listening at port %s", type, port); + + base_priority = GetThreadPriority(GetCurrentThread()); do { @@ -95,6 +99,8 @@ static DWORD WINAPI RM_EntryPoint(void* param) ShowMessage("RM%d: Client connected", type); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(type)); + switch (type) { case ResearchModeSensorType::LEFT_FRONT: RM_Stream(sensor, clientsocket, locator); break; @@ -108,6 +114,8 @@ static DWORD WINAPI RM_EntryPoint(void* param) case ResearchModeSensorType::IMU_MAG: RM_Stream(sensor, clientsocket, locator); break; } + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("RM%d: Client disconnected", type); @@ -144,7 +152,7 @@ void RM_Cleanup() { WaitForMultipleObjects((DWORD)g_threads.size(), g_threads.data(), TRUE, INFINITE); - for (auto thread : g_threads) { CloseHandle(thread); } + for (auto const& thread : g_threads) { CloseHandle(thread); } CloseHandle(g_event_quit); g_threads.clear(); diff --git a/hl2ss/hl2ss/stream_rm_vlc.cpp b/hl2ss/hl2ss/stream_rm_vlc.cpp index 7452509a4..31c1ce2e1 100644 --- a/hl2ss/hl2ss/stream_rm_vlc.cpp +++ b/hl2ss/hl2ss/stream_rm_vlc.cpp @@ -16,11 +16,21 @@ using namespace winrt::Windows::Foundation::Numerics; using namespace winrt::Windows::Perception; using namespace winrt::Windows::Perception::Spatial; +struct VLC_Metadata +{ + uint64_t sensor_ticks; + uint64_t exposure; + uint32_t gain; + uint32_t _reserved; + uint64_t timestamp; + float4x4 pose; +}; + //----------------------------------------------------------------------------- // Global Variables //----------------------------------------------------------------------------- -static float4x4 g_pose_sh; +static VLC_Metadata g_pose_sh[4]; //----------------------------------------------------------------------------- // Functions @@ -32,12 +42,12 @@ static void RM_VLC_TranslateEncoderOptions(std::vector const& options, exposure_factor = 0.0; constant_factor = 0; - for (int i = 0; i < (int)(options.size() / 2); ++i) + for (int i = 0; i < (int)(options.size() & ~1ULL); i += 2) { - switch (options[2 * i]) + switch (options[i]) { - case 0xFFFFFFFFFFFFFFFE: constant_factor = (int64_t)options[(2 * i) + 1]; break; - case 0xFFFFFFFFFFFFFFFF: exposure_factor = *(double*)&options[(2 * i) + 1]; break; + case 0xFFFFFFFFFFFFFFFE: constant_factor = (int64_t)options[i + 1]; break; + case 0xFFFFFFFFFFFFFFFF: exposure_factor = *(double*)&options[i + 1]; break; } } } @@ -50,41 +60,40 @@ void RM_VLC_SendSample(IMFSample* pSample, void* param) LONGLONG sampletime; BYTE* pBytes; DWORD cbData; - WSABUF wsaBuf[ENABLE_LOCATION ? 4 : 3]; - HookCallbackSocket* user; - H26xFormat* format; - bool sh; - bool ok; + WSABUF wsaBuf[ENABLE_LOCATION ? 5 : 4]; - user = (HookCallbackSocket*)param; - format = (H26xFormat*)user->format; - sh = format->profile != H26xProfile::H26xProfile_None; + HookCallbackSocket* user = (HookCallbackSocket*)param; + H26xFormat* format = (H26xFormat*)user->format; + bool sh = format->profile != H26xProfile::H26xProfile_None; + int id = format->framerate; pSample->GetSampleTime(&sampletime); pSample->ConvertToContiguousBuffer(&pBuffer); + if (!sh) { pSample->GetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&g_pose_sh[id], sizeof(g_pose_sh[id]), NULL); } + pBuffer->Lock(&pBytes, NULL, &cbData); - pack_buffer(wsaBuf, 0, &sampletime, sizeof(sampletime)); - pack_buffer(wsaBuf, 1, &cbData, sizeof(cbData)); + int const metadata = sizeof(g_pose_sh[id]) - sizeof(g_pose_sh[id].timestamp) - sizeof(g_pose_sh[id].pose); + DWORD cbDataEx = cbData + metadata; + + pack_buffer(wsaBuf, 0, &g_pose_sh[id].timestamp, sizeof(g_pose_sh[id].timestamp)); + pack_buffer(wsaBuf, 1, &cbDataEx, sizeof(cbDataEx)); pack_buffer(wsaBuf, 2, pBytes, cbData); + pack_buffer(wsaBuf, 3, &g_pose_sh[id], metadata); if constexpr(ENABLE_LOCATION) { - if (!sh) { pSample->GetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&g_pose_sh, sizeof(g_pose_sh), NULL); } - pack_buffer(wsaBuf, 3, &g_pose_sh, sizeof(g_pose_sh)); + pack_buffer(wsaBuf, 4, &g_pose_sh[id].pose, sizeof(g_pose_sh[id].pose)); } - ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); + bool ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); if (!ok) { SetEvent(user->clientevent); } pBuffer->Unlock(); pBuffer->Release(); - if constexpr (ENABLE_LOCATION) - { - if (sh) { pSample->GetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&g_pose_sh, sizeof(g_pose_sh), NULL); } - } + if (sh) { pSample->GetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&g_pose_sh[id], sizeof(g_pose_sh[id]), NULL); } } // OK @@ -99,7 +108,6 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca PerceptionTimestamp ts = nullptr; uint32_t f = 0; - float4x4 pose; IResearchModeSensorFrame* pSensorFrame; // Release ResearchModeSensorTimestamp timestamp; IResearchModeSensorVLCFrame* pVLCFrame; // Release @@ -119,10 +127,11 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca uint32_t framebytes; HRESULT hr; VideoSubtype subtype; - UINT64 exposure; double exposure_factor; int64_t constant_factor; UINT64 adjusted_timestamp; + VLC_Metadata vlc_metadata; + int id; bool ok; ok = ReceiveH26xFormat_Divisor(clientsocket, format); @@ -134,6 +143,8 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca ok = ReceiveH26xEncoder_Options(clientsocket, options); if (!ok) { return; } + id = sensor->GetSensorType(); + format.width = width; format.height = height; format.framerate = framerate; @@ -152,10 +163,13 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca CreateSinkWriterVideo(&pSink, &pSinkWriter, &dwVideoIndex, subtype, format, options, RM_VLC_SendSample, &user); + format.framerate = (uint8_t)id; + RM_VLC_TranslateEncoderOptions(options, exposure_factor, constant_factor); framebytes = lumasize + chromasize; - memset(&g_pose_sh, 0, sizeof(g_pose_sh)); + + memset(&g_pose_sh[id], 0, sizeof(g_pose_sh[id])); sensor->OpenStream(); @@ -170,7 +184,8 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca pSensorFrame->QueryInterface(IID_PPV_ARGS(&pVLCFrame)); pVLCFrame->GetBuffer(&pImage, &length); - pVLCFrame->GetExposure(&exposure); + pVLCFrame->GetExposure(&vlc_metadata.exposure); + pVLCFrame->GetGain(&vlc_metadata.gain); MFCreateMemoryBuffer(framebytes, &pBuffer); @@ -182,8 +197,10 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca MFCreateSample(&pSample); - adjusted_timestamp = timestamp.HostTicks + (int64_t)((exposure_factor * exposure) / 100.0) + constant_factor; - + adjusted_timestamp = timestamp.HostTicks + (int64_t)((exposure_factor * vlc_metadata.exposure) / 100.0) + constant_factor; + vlc_metadata.timestamp = adjusted_timestamp; + vlc_metadata.sensor_ticks = timestamp.SensorTicks; + pSample->AddBuffer(pBuffer); pSample->SetSampleDuration(duration); pSample->SetSampleTime(adjusted_timestamp); @@ -191,10 +208,11 @@ void RM_VLC_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca if constexpr (ENABLE_LOCATION) { ts = QPCTimestampToPerceptionTimestamp(adjusted_timestamp); - pose = Locator_Locate(ts, locator, Locator_GetWorldCoordinateSystem(ts)); - pSample->SetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&pose, sizeof(float4x4)); + vlc_metadata.pose = Locator_Locate(ts, locator, Locator_GetWorldCoordinateSystem(ts)); } + pSample->SetBlob(MF_USER_DATA_PAYLOAD, (UINT8*)&vlc_metadata, sizeof(vlc_metadata)); + pSinkWriter->WriteSample(dwVideoIndex, pSample); pSample->Release(); diff --git a/hl2ss/hl2ss/stream_rm_zht.cpp b/hl2ss/hl2ss/stream_rm_zht.cpp index 70371e48b..e4d8f4be6 100644 --- a/hl2ss/hl2ss/stream_rm_zht.cpp +++ b/hl2ss/hl2ss/stream_rm_zht.cpp @@ -32,6 +32,8 @@ typedef void(*RM_ZXX_KERNEL)(zdepth::DepthCompressor&, uint16_t const*, std::vec struct RM_ZAB_Blob { std::vector* z; + uint64_t sensor_ticks; + uint64_t timestamp; float4x4 pose; }; @@ -112,7 +114,7 @@ void RM_ZHT_SendSample(IMFSample* pSample, void* param) uint32_t size_z; uint32_t size_ab; bool has_z; - WSABUF wsaBuf[ENABLE_LOCATION ? 7 : 6]; + WSABUF wsaBuf[ENABLE_LOCATION ? 8 : 7]; HookCallbackSocket* user; H26xFormat* format; bool sh; @@ -135,18 +137,19 @@ void RM_ZHT_SendSample(IMFSample* pSample, void* param) pBuffer->Lock(&pBytes, NULL, &cbData); size_ab = (uint32_t)cbData; - size_p = sizeof(size_z) + sizeof(size_ab) + size_z + size_ab; + size_p = sizeof(size_z) + sizeof(size_ab) + size_z + size_ab + sizeof(g_blob_sh.sensor_ticks); - pack_buffer(wsaBuf, 0, &sampletime, sizeof(sampletime)); + pack_buffer(wsaBuf, 0, &g_blob_sh.timestamp, sizeof(g_blob_sh.timestamp)); pack_buffer(wsaBuf, 1, &size_p, sizeof(size_p)); pack_buffer(wsaBuf, 2, &size_z, sizeof(size_z)); pack_buffer(wsaBuf, 3, &size_ab, sizeof(size_ab)); pack_buffer(wsaBuf, 4, p_z, size_z); pack_buffer(wsaBuf, 5, pBytes, size_ab); + pack_buffer(wsaBuf, 6, &g_blob_sh.sensor_ticks, sizeof(g_blob_sh.sensor_ticks)); if constexpr (ENABLE_LOCATION) { - pack_buffer(wsaBuf, 6, &g_blob_sh.pose, sizeof(g_blob_sh.pose)); + pack_buffer(wsaBuf, 7, &g_blob_sh.pose, sizeof(g_blob_sh.pose)); } ok = send_multiple(user->clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); @@ -263,6 +266,9 @@ void RM_ZHT_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca kernel_blob(compressor, pDepth, blob.z); + blob.timestamp = timestamp.HostTicks; + blob.sensor_ticks = timestamp.SensorTicks; + if constexpr (ENABLE_LOCATION) { ts = QPCTimestampToPerceptionTimestamp(timestamp.HostTicks); diff --git a/hl2ss/hl2ss/stream_rm_zlt.cpp b/hl2ss/hl2ss/stream_rm_zlt.cpp index 9ce2d476e..4992ed0fa 100644 --- a/hl2ss/hl2ss/stream_rm_zlt.cpp +++ b/hl2ss/hl2ss/stream_rm_zlt.cpp @@ -55,7 +55,7 @@ void RM_ZLT_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca BitmapPropertySet pngProperties; ZABFormat zabFormat; uint32_t streamSize; - WSABUF wsaBuf[ENABLE_LOCATION ? 4 : 3]; + WSABUF wsaBuf[ENABLE_LOCATION ? 5 : 4]; HRESULT hr; H26xFormat format; bool ok; @@ -84,35 +84,38 @@ void RM_ZLT_Stream(IResearchModeSensor* sensor, SOCKET clientsocket, SpatialLoca pDepthFrame->GetBuffer(&pDepth, &nDepthCount); pDepthFrame->GetAbDepthBuffer(&pAbImage, &nAbCount); - auto softwareBitmap = SoftwareBitmap(BitmapPixelFormat::Bgra8, width, height, BitmapAlphaMode::Straight); + auto const& softwareBitmap = SoftwareBitmap(BitmapPixelFormat::Bgra8, width, height, BitmapAlphaMode::Straight); { - auto bitmapBuffer = softwareBitmap.LockBuffer(BitmapBufferAccessMode::Write); - auto spMemoryBufferByteAccess = bitmapBuffer.CreateReference().as(); + auto const& bitmapBuffer = softwareBitmap.LockBuffer(BitmapBufferAccessMode::Write); + auto const& spMemoryBufferByteAccess = bitmapBuffer.CreateReference().as(); spMemoryBufferByteAccess->GetBuffer(&pixelBufferData, &pixelBufferDataLength); Neon_ZLTToBGRA8(pSigma, pDepth, pAbImage, (u32*)pixelBufferData); } - auto stream = InMemoryRandomAccessStream(); - auto encoder = BitmapEncoder::CreateAsync(BitmapEncoder::PngEncoderId(), stream, pngProperties).get(); + auto const& stream = InMemoryRandomAccessStream(); + auto const& encoder = BitmapEncoder::CreateAsync(BitmapEncoder::PngEncoderId(), stream, pngProperties).get(); encoder.SetSoftwareBitmap(softwareBitmap); encoder.FlushAsync().get(); streamSize = (uint32_t)stream.Size(); - auto streamBuf = Buffer(streamSize); + auto const& streamBuf = Buffer(streamSize); stream.ReadAsync(streamBuf, streamSize, InputStreamOptions::None).get(); + DWORD streamSizeEx = streamSize + sizeof(timestamp.SensorTicks); + pack_buffer(wsaBuf, 0, ×tamp.HostTicks, sizeof(timestamp.HostTicks)); - pack_buffer(wsaBuf, 1, &streamSize, sizeof(streamSize)); + pack_buffer(wsaBuf, 1, &streamSizeEx, sizeof(streamSizeEx)); pack_buffer(wsaBuf, 2, streamBuf.data(), streamSize); + pack_buffer(wsaBuf, 3, ×tamp.SensorTicks, sizeof(timestamp.SensorTicks)); if constexpr (ENABLE_LOCATION) { ts = QPCTimestampToPerceptionTimestamp(timestamp.HostTicks); pose = Locator_Locate(ts, locator, Locator_GetWorldCoordinateSystem(ts)); - pack_buffer(wsaBuf, 3, &pose, sizeof(pose)); + pack_buffer(wsaBuf, 4, &pose, sizeof(pose)); } ok = send_multiple(clientsocket, wsaBuf, sizeof(wsaBuf) / sizeof(WSABUF)); diff --git a/hl2ss/hl2ss/stream_si.cpp b/hl2ss/hl2ss/stream_si.cpp index 3c94d9a6a..93a437c18 100644 --- a/hl2ss/hl2ss/stream_si.cpp +++ b/hl2ss/hl2ss/stream_si.cpp @@ -3,6 +3,7 @@ #include "server.h" #include "locator.h" #include "spatial_input.h" +#include "extended_execution.h" #include "ports.h" #include "timestamps.h" #include "log.h" @@ -82,6 +83,7 @@ static DWORD WINAPI SI_EntryPoint(void *param) SOCKET listensocket; // closesocket SOCKET clientsocket; // closesocket + int base_priority; ShowMessage("SI: Waiting for consent"); @@ -91,6 +93,8 @@ static DWORD WINAPI SI_EntryPoint(void *param) ShowMessage("SI: Listening at port %s", PORT_NAME_SI); + base_priority = GetThreadPriority(GetCurrentThread()); + do { ShowMessage("SI: Waiting for client"); @@ -100,8 +104,12 @@ static DWORD WINAPI SI_EntryPoint(void *param) ShowMessage("SI: Client connected"); + SetThreadPriority(GetCurrentThread(), ExtendedExecution_GetInterfacePriority(PORT_NUMBER_SI - PORT_NUMBER_BASE)); + SI_Stream(clientsocket); + SetThreadPriority(GetCurrentThread(), base_priority); + closesocket(clientsocket); ShowMessage("SI: Client disconnected"); diff --git a/hl2ss/hl2ss/types.h b/hl2ss/hl2ss/types.h index 3ce4be0ec..124f8a67a 100644 --- a/hl2ss/hl2ss/types.h +++ b/hl2ss/hl2ss/types.h @@ -17,3 +17,18 @@ union v8 { u8 b; s8 c; }; union v16 { struct { v8 b0, b1; } b; u16 w; s16 s; }; union v32 { struct { v16 w0, w1; } w; u32 d; s32 i; }; union v64 { struct { v32 d0, d1; } d; u64 q; s64 l; }; + +constexpr uint32_t bit(uint32_t b) +{ + return (1UL << b); +} + +constexpr bool bit_test(uint32_t v, uint32_t b) +{ + return v & bit(b); +} + +constexpr uint32_t bit_field(uint32_t v, uint32_t b, uint32_t mask) +{ + return (v >> b) & mask; +} diff --git a/hl2ss/hl2ss/voice_input.cpp b/hl2ss/hl2ss/voice_input.cpp index 54d9a6f7d..eaefa4b14 100644 --- a/hl2ss/hl2ss/voice_input.cpp +++ b/hl2ss/hl2ss/voice_input.cpp @@ -36,7 +36,7 @@ void VoiceInput_Initialize() static uint32_t VoiceInput_FindID(winrt::hstring const& query) { uint32_t index = 0; - for (auto& command : g_commands) { if (command == query) { return index; } else { ++index; } } + for (auto const& command : g_commands) { if (command == query) { return index; } else { ++index; } } return index; } diff --git a/hl2ss_unity/Assets/Plugins/WSA/hl2ss.dll b/hl2ss_unity/Assets/Plugins/WSA/hl2ss.dll index 4fb39420f..d07004868 100644 Binary files a/hl2ss_unity/Assets/Plugins/WSA/hl2ss.dll and b/hl2ss_unity/Assets/Plugins/WSA/hl2ss.dll differ diff --git a/hl2ss_unreal/Plugins/hl2ss/Binaries/hl2ss/hl2ss.dll b/hl2ss_unreal/Plugins/hl2ss/Binaries/hl2ss/hl2ss.dll index 4fb39420f..d07004868 100644 Binary files a/hl2ss_unreal/Plugins/hl2ss/Binaries/hl2ss/hl2ss.dll and b/hl2ss_unreal/Plugins/hl2ss/Binaries/hl2ss/hl2ss.dll differ diff --git a/viewer/client_ipc_rc.py b/viewer/client_ipc_rc.py index 2a1883483..2bcedd44d 100644 --- a/viewer/client_ipc_rc.py +++ b/viewer/client_ipc_rc.py @@ -65,6 +65,9 @@ # https://learn.microsoft.com/en-us/uwp/api/windows.media.devices.scenemodecontrol?view=winrt-22621 scene_mode = hl2ss.PV_CaptureSceneMode.Auto +# Set eye selection +eye_selection = False + #------------------------------------------------------------------------------ client = hl2ss_lnm.ipc_rc(host, hl2ss.IPCPort.REMOTE_CONFIGURATION) @@ -93,6 +96,9 @@ client.set_pv_iso_speed(iso_speed_mode, iso_speed_value) client.set_pv_backlight_compensation(backlight_compensation_state) client.set_pv_scene_mode(scene_mode) + client.set_flat_mode(flat_mode) +client.set_rm_eye_selection(eye_selection) + client.close() diff --git a/viewer/client_stream_pv.py b/viewer/client_stream_pv.py index 6c39b0403..0110c4808 100644 --- a/viewer/client_stream_pv.py +++ b/viewer/client_stream_pv.py @@ -44,8 +44,9 @@ # Effective FPS is framerate / divisor divisor = 1 -# Video encoding profile +# Video encoding profile and bitrate (None = default) profile = hl2ss.VideoProfile.H265_MAIN +bitrate = None # Decoded format # Options include: @@ -73,6 +74,8 @@ print(data.intrinsics) print('RigNode Extrinsics') print(data.extrinsics) + print(f'Intrinsics MF: {data.intrinsics_mf}') + print(f'Extrinsics MF: {data.extrinsics_mf}') else: enable = True @@ -84,16 +87,25 @@ def on_press(key): listener = keyboard.Listener(on_press=on_press) listener.start() - client = hl2ss_lnm.rx_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, mode=mode, width=width, height=height, framerate=framerate, divisor=divisor, profile=profile, decoded_format=decoded_format) + client = hl2ss_lnm.rx_pv(host, hl2ss.StreamPort.PERSONAL_VIDEO, mode=mode, width=width, height=height, framerate=framerate, divisor=divisor, profile=profile, bitrate=bitrate, decoded_format=decoded_format) client.open() while (enable): data = client.get_next_packet() - print(f'Pose at time {data.timestamp}') - print(data.pose) + print(f'Frame captured at {data.timestamp}') print(f'Focal length: {data.payload.focal_length}') print(f'Principal point: {data.payload.principal_point}') + print(f'Exposure Time: {data.payload.exposure_time}') + print(f'Exposure Compensation: {data.payload.exposure_compensation}') + print(f'Lens Position (Focus): {data.payload.lens_position}') + print(f'Focus State: {data.payload.focus_state}') + print(f'ISO Speed: {data.payload.iso_speed}') + print(f'White Balance: {data.payload.white_balance}') + print(f'ISO Gains: {data.payload.iso_gains}') + print(f'White Balance Gains: {data.payload.white_balance_gains}') + print(f'Pose') + print(data.pose) cv2.imshow('Video', data.payload.image) cv2.waitKey(1) diff --git a/viewer/client_stream_rm_depth_ahat.py b/viewer/client_stream_rm_depth_ahat.py index 7350f3c57..ca767b9aa 100644 --- a/viewer/client_stream_rm_depth_ahat.py +++ b/viewer/client_stream_rm_depth_ahat.py @@ -32,7 +32,7 @@ # Effective framerate is framerate / divisor divisor = 1 -# Depth encoding profile +# Depth encoding profile, AB encoding profile and bitrate (None = default) # SAME: use same compression as AB # AB RAW: # - data streamed as-is (most accurate) @@ -45,10 +45,9 @@ # - increased minimum range (objects close to the camera get truncated) # - full framerate # - requires building the pyzdepth extension (one time only) -profile_z = hl2ss.DepthProfile.SAME - -# AB encoding profile +profile_z = hl2ss.DepthProfile.SAME profile_ab = hl2ss.VideoProfile.H265_MAIN +bitrate = None #------------------------------------------------------------------------------ @@ -82,9 +81,11 @@ def on_press(key): while (enable): data = client.get_next_packet() - - print(f'Pose at time {data.timestamp}') - print(data.pose) + + print(f'Frame captured at {data.timestamp}') + print(f'Sensor Ticks: {data.payload.sensor_ticks}') + print(f'Pose') + print(data.pose) cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Scaled for visibility cv2.imshow('AB', data.payload.ab / np.max(data.payload.ab)) # Scaled for visibility diff --git a/viewer/client_stream_rm_depth_longthrow.py b/viewer/client_stream_rm_depth_longthrow.py index 5c57f9705..107795116 100644 --- a/viewer/client_stream_rm_depth_longthrow.py +++ b/viewer/client_stream_rm_depth_longthrow.py @@ -62,7 +62,9 @@ def on_press(key): while (enable): data = client.get_next_packet() - print(f'Pose at time {data.timestamp}') + print(f'Frame captured at {data.timestamp}') + print(f'Sensor Ticks: {data.payload.sensor_ticks}') + print(f'Pose') print(data.pose) cv2.imshow('Depth', data.payload.depth / np.max(data.payload.depth)) # Scaled for visibility diff --git a/viewer/client_stream_rm_imu.py b/viewer/client_stream_rm_imu.py index a8d919487..36bbc1646 100644 --- a/viewer/client_stream_rm_imu.py +++ b/viewer/client_stream_rm_imu.py @@ -58,15 +58,14 @@ def on_press(key): while (enable): data = client.get_next_packet() - print(f'Pose at time {data.timestamp}') - print(data.pose) - imu_data = hl2ss.unpack_rm_imu(data.payload) count = imu_data.get_count() sample = imu_data.get_frame(0) print(f'Got {count} samples at time {data.timestamp}') print(f'First sample: sensor_ticks={sample.vinyl_hup_ticks} soc_ticks={sample.soc_ticks} x={sample.x} y={sample.y} z={sample.z} temperature={sample.temperature}') + print(f'Pose') + print(data.pose) client.close() listener.join() diff --git a/viewer/client_stream_rm_vlc.py b/viewer/client_stream_rm_vlc.py index c1019b8b7..6a019db1f 100644 --- a/viewer/client_stream_rm_vlc.py +++ b/viewer/client_stream_rm_vlc.py @@ -36,8 +36,9 @@ # Effective framerate is framerate / divisor divisor = 1 -# Video encoding profile +# Video encoding profile and bitrate (None = default) profile = hl2ss.VideoProfile.H265_MAIN +bitrate = None #------------------------------------------------------------------------------ @@ -69,9 +70,15 @@ def on_press(key): while (enable): data = client.get_next_packet() - print(f'Pose at time {data.timestamp}') + + print(f'Frame captured at {data.timestamp}') + print(f'Sensor Ticks: {data.payload.sensor_ticks}') + print(f'Exposure: {data.payload.exposure}') + print(f'Gain: {data.payload.gain}') + print(f'Pose') print(data.pose) - cv2.imshow('Video', data.payload) + + cv2.imshow('Video', data.payload.image) cv2.waitKey(1) client.close() diff --git a/viewer/hl2ss.py b/viewer/hl2ss.py index 716e4e1dd..89b3b48b3 100644 --- a/viewer/hl2ss.py +++ b/viewer/hl2ss.py @@ -876,6 +876,25 @@ def get_audio_codec_bitrate(profile): # RM VLC Decoder #------------------------------------------------------------------------------ +class _RM_VLC_Frame: + def __init__(self, image, sensor_ticks, exposure, gain): + self.image = image + self.sensor_ticks = sensor_ticks + self.exposure = exposure + self.gain = gain + + +def unpack_rm_vlc(payload): + image = payload[:-24] + metadata = payload[-24:] + + sensor_ticks = np.frombuffer(metadata, dtype=np.uint64, offset=0, count=1) + exposure = np.frombuffer(metadata, dtype=np.uint64, offset=8, count=1) + gain = np.frombuffer(metadata, dtype=np.uint32, offset=16, count=1) + + return _RM_VLC_Frame(image, sensor_ticks, exposure, gain) + + class _decode_rm_vlc: def __init__(self, profile): self.profile = profile @@ -907,9 +926,10 @@ def decode_rm_vlc(profile): #------------------------------------------------------------------------------ class _RM_Depth_Frame: - def __init__(self, depth, ab): - self.depth = depth - self.ab = ab + def __init__(self, depth, ab, sensor_ticks): + self.depth = depth + self.ab = ab + self.sensor_ticks = sensor_ticks class _Mode0Layout_RM_DEPTH_AHAT_STRUCT: @@ -925,7 +945,7 @@ class _Mode0Layout_RM_DEPTH_AHAT: END_AB_V_Y = BEGIN_AB_V_Y + (Parameters_RM_DEPTH_AHAT.WIDTH // 4) -def _unpack_rm_depth_ahat_nv12_as_yuv420p(yuv): +def _unpack_rm_depth_ahat_nv12_as_yuv420p(yuv, sensor_ticks): y = yuv[_Mode0Layout_RM_DEPTH_AHAT.BEGIN_DEPTH_Y : _Mode0Layout_RM_DEPTH_AHAT.END_DEPTH_Y, :] u = yuv[_Mode0Layout_RM_DEPTH_AHAT.BEGIN_AB_U_Y : _Mode0Layout_RM_DEPTH_AHAT.END_AB_U_Y, :].reshape((Parameters_RM_DEPTH_AHAT.HEIGHT, Parameters_RM_DEPTH_AHAT.WIDTH // 4)) v = yuv[_Mode0Layout_RM_DEPTH_AHAT.BEGIN_AB_V_Y : _Mode0Layout_RM_DEPTH_AHAT.END_AB_V_Y, :].reshape((Parameters_RM_DEPTH_AHAT.HEIGHT, Parameters_RM_DEPTH_AHAT.WIDTH // 4)) @@ -941,7 +961,7 @@ def _unpack_rm_depth_ahat_nv12_as_yuv420p(yuv): ab[:, 2::4] = v ab[:, 3::4] = v - return _RM_Depth_Frame(depth, ab) + return _RM_Depth_Frame(depth, ab, sensor_ticks) class _decode_rm_depth_ahat: @@ -952,9 +972,9 @@ def create(self): self._codec = av.CodecContext.create(get_video_codec_name(self.profile), 'r') def decode(self, payload): - for packet in self._codec.parse(payload[_Mode0Layout_RM_DEPTH_AHAT_STRUCT.BASE:]): + for packet in self._codec.parse(payload[_Mode0Layout_RM_DEPTH_AHAT_STRUCT.BASE:-8]): for frame in self._codec.decode(packet): - return _unpack_rm_depth_ahat_nv12_as_yuv420p(frame.to_ndarray()) + return _unpack_rm_depth_ahat_nv12_as_yuv420p(frame.to_ndarray(), np.frombuffer(payload[-8:], dtype=np.uint64, offset=0, count=1)) return None @@ -963,9 +983,10 @@ def create(self): pass def decode(self, payload): - depth = np.frombuffer(payload, dtype=np.uint16, offset=_Mode0Layout_RM_DEPTH_AHAT_STRUCT.BASE, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) - ab = np.frombuffer(payload, dtype=np.uint16, offset=_Mode0Layout_RM_DEPTH_AHAT_STRUCT.BASE + Parameters_RM_DEPTH_AHAT.PIXELS * _SIZEOF.WORD, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) - return _RM_Depth_Frame(depth, ab) + depth = np.frombuffer(payload, dtype=np.uint16, offset=_Mode0Layout_RM_DEPTH_AHAT_STRUCT.BASE, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) + ab = np.frombuffer(payload, dtype=np.uint16, offset=_Mode0Layout_RM_DEPTH_AHAT_STRUCT.BASE + Parameters_RM_DEPTH_AHAT.PIXELS * _SIZEOF.WORD, count=Parameters_RM_DEPTH_AHAT.PIXELS).reshape(Parameters_RM_DEPTH_AHAT.SHAPE) + sensor_ticks = np.frombuffer(payload[-8:], dtype=np.uint64, offset=0, count=1) + return _RM_Depth_Frame(depth, ab, sensor_ticks) class _decompress_zdepth: @@ -1019,10 +1040,11 @@ def decode(self, payload): start_ab = end_z end_ab = start_ab + size_ab - depth = self._codec_z.decode(payload[start_z:end_z]) - ab = self._codec_ab.decode(payload[start_ab:end_ab]) + depth = self._codec_z.decode(payload[start_z:end_z]) + ab = self._codec_ab.decode(payload[start_ab:end_ab]) + sensor_ticks = np.frombuffer(payload[-8:], dtype=np.uint64, offset=0, count=1) - return _RM_Depth_Frame(depth, ab) + return _RM_Depth_Frame(depth, ab, sensor_ticks) def decode_rm_depth_ahat(profile_z, profile_ab): @@ -1030,10 +1052,11 @@ def decode_rm_depth_ahat(profile_z, profile_ab): def decode_rm_depth_longthrow(payload): - composite = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_UNCHANGED) - h, w, _ = composite.shape - image = composite.view(np.uint16).reshape((2*h, w)) - return _RM_Depth_Frame(image[:h, :], image[h:, :]) + composite = cv2.imdecode(np.frombuffer(payload[:-8], dtype=np.uint8), cv2.IMREAD_UNCHANGED) + h, w, _ = composite.shape + image = composite.view(np.uint16).reshape((2*h, w)) + sensor_ticks = np.frombuffer(payload[-8:], dtype=np.uint64, offset=0, count=1) + return _RM_Depth_Frame(image[:h, :], image[h:, :], sensor_ticks) #------------------------------------------------------------------------------ @@ -1067,11 +1090,27 @@ def get_frame(self, index): # PV Decoder #------------------------------------------------------------------------------ +class PV_FocusState: + UNINITIALIZED = 0 + LOST = 1 + SEARCHING = 2 + FOCUSED = 3 + FAILED = 4 + + class _PV_Frame: - def __init__(self, image, focal_length, principal_point): - self.image = image - self.focal_length = focal_length - self.principal_point = principal_point + def __init__(self, image, focal_length, principal_point, exposure_time, exposure_compensation, lens_position, focus_state, iso_speed, white_balance, iso_gains, white_balance_gains): + self.image = image + self.focal_length = focal_length + self.principal_point = principal_point + self.exposure_time = exposure_time + self.exposure_compensation = exposure_compensation + self.lens_position = lens_position + self.focus_state = focus_state + self.iso_speed = iso_speed + self.white_balance = white_balance + self.iso_gains = iso_gains + self.white_balance_gains = white_balance_gains def create_pv_intrinsics(focal_length, principal_point): @@ -1091,7 +1130,21 @@ def update_pv_intrinsics(intrinsics, focal_length, principal_point): def unpack_pv(payload): - return _PV_Frame(payload[:-16], np.frombuffer(payload[-16:-8], dtype=np.float32), np.frombuffer(payload[-8:], dtype=np.float32)) + image = payload[0:-80] + metadata = payload[-80:] + + focal_length = np.frombuffer(metadata, dtype=np.float32, offset=0, count=2) + principal_point = np.frombuffer(metadata, dtype=np.float32, offset=8, count=2) + exposure_time = np.frombuffer(metadata, dtype=np.uint64, offset=16, count=1) + exposure_compensation = np.frombuffer(metadata, dtype=np.uint64, offset=24, count=2) + lens_position = np.frombuffer(metadata, dtype=np.uint32, offset=40, count=1) + focus_state = np.frombuffer(metadata, dtype=np.uint32, offset=44, count=1) + iso_speed = np.frombuffer(metadata, dtype=np.uint32, offset=48, count=1) + white_balance = np.frombuffer(metadata, dtype=np.uint32, offset=52, count=1) + iso_gains = np.frombuffer(metadata, dtype=np.float32, offset=56, count=2) + white_balance_gains = np.frombuffer(metadata, dtype=np.float32, offset=64, count=3) + + return _PV_Frame(image, focal_length, principal_point, exposure_time, exposure_compensation, lens_position, focus_state, iso_speed, white_balance, iso_gains, white_balance_gains) def get_video_stride(width): @@ -1373,7 +1426,8 @@ def open(self): def get_next_packet(self): data = super().get_next_packet() - data.payload = self._codec.decode(data.payload) + data.payload = unpack_rm_vlc(data.payload) + data.payload.image = self._codec.decode(data.payload.image) return data def close(self): @@ -1549,7 +1603,11 @@ class _Mode2Layout_PV: END_PROJECTION = BEGIN_PROJECTION + 16 BEGIN_EXTRINSICS = END_PROJECTION END_EXTRINSICS = BEGIN_EXTRINSICS + 16 - FLOAT_COUNT = 2 + 2 + 3 + 2 + 16 + 16 + BEGIN_INTRINSICS_MF = END_EXTRINSICS + END_INTRINSICS_MF = BEGIN_INTRINSICS_MF + 4 + BEGIN_EXTRINSICS_MF = END_INTRINSICS_MF + END_EXTRINSICS_MF = BEGIN_EXTRINSICS_MF + 7 + FLOAT_COUNT = 2 + 2 + 3 + 2 + 16 + 16 + 4 + 7 class _Mode2_RM_VLC: @@ -1585,7 +1643,7 @@ def __init__(self, extrinsics): class _Mode2_PV: - def __init__(self, focal_length, principal_point, radial_distortion, tangential_distortion, projection, intrinsics, extrinsics): + def __init__(self, focal_length, principal_point, radial_distortion, tangential_distortion, projection, intrinsics, extrinsics, intriniscs_mf, extrinsics_mf): self.focal_length = focal_length self.principal_point = principal_point self.radial_distortion = radial_distortion @@ -1593,6 +1651,8 @@ def __init__(self, focal_length, principal_point, radial_distortion, tangential_ self.projection = projection self.intrinsics = intrinsics self.extrinsics = extrinsics + self.intrinsics_mf = intriniscs_mf + self.extrinsics_mf = extrinsics_mf def _download_mode2_data(host, port, configuration, bytes): @@ -1676,10 +1736,12 @@ def download_calibration_pv(host, port, width, height, framerate): tangential_distortion = floats[_Mode2Layout_PV.BEGIN_TANGENTIALDISTORTION : _Mode2Layout_PV.END_TANGENTIALDISTORTION] projection = floats[_Mode2Layout_PV.BEGIN_PROJECTION : _Mode2Layout_PV.END_PROJECTION ].reshape((4, 4)) extrinsics = floats[_Mode2Layout_PV.BEGIN_EXTRINSICS : _Mode2Layout_PV.END_EXTRINSICS ].reshape((4, 4)) + intrinsics_mf = floats[_Mode2Layout_PV.BEGIN_INTRINSICS_MF : _Mode2Layout_PV.END_INTRINSICS_MF ] + extrinsics_mf = floats[_Mode2Layout_PV.BEGIN_EXTRINSICS_MF : _Mode2Layout_PV.END_EXTRINSICS_MF ] intrinsics = np.array([[-focal_length[0], 0, 0, 0], [0, focal_length[1], 0, 0], [principal_point[0], principal_point[1], 1, 0], [0, 0, 0, 1]], dtype=np.float32) - return _Mode2_PV(focal_length, principal_point, radial_distortion, tangential_distortion, projection, intrinsics, extrinsics) + return _Mode2_PV(focal_length, principal_point, radial_distortion, tangential_distortion, projection, intrinsics, extrinsics, intrinsics_mf, extrinsics_mf) def download_devicelist_extended_audio(host, port): @@ -1819,6 +1881,11 @@ class PV_IsoSpeedValue: Max = 3200 +class PV_BacklightCompensationState: + Disable = 0 + Enable = 1 + + class PV_CaptureSceneMode: Auto = 0 Macro = 2 @@ -1834,11 +1901,44 @@ class PV_CaptureSceneMode: Backlit = 12 -class PV_BacklightCompensationState: - Disable = 0 - Enable = 1 +class PV_MediaCaptureOptimization: + Default = 0 + Quality = 1 + Latency = 2 + Power = 3 + LatencyThenQuality = 4 + LatencyThenPower = 5 + PowerAndQuality = 6 +class PV_CaptureUse: + NotSet = 0 + Photo = 1 + Video = 2 + + +class PV_OpticalImageStabilizationMode: + Off = 0 + On = 1 + + +class PV_HdrVideoMode: + Off = 0 + On = 1 + Auto = 2 + + +class PV_RegionOfInterestType: + Unknown = 0 + Face = 1 + +class InterfacePriority: + LOWEST = -2 + BELOW_NORMAL = -1 + NORMAL = 0 + ABOVE_NORMAL = 1 + HIGHEST = 2 + class ipc_rc(_context_manager): _CMD_GET_APPLICATION_VERSION = 0x00 _CMD_GET_UTC_OFFSET = 0x01 @@ -1854,6 +1954,13 @@ class ipc_rc(_context_manager): _CMD_SET_PV_BACKLIGHT_COMPENSATION = 0x0B _CMD_SET_PV_SCENE_MODE = 0x0C _CMD_SET_FLAT_MODE = 0x0D + _CMD_SET_RM_EYE_SELECTION = 0x0E + _CMD_SET_PV_DESIRED_OPTIMIZATION = 0x0F + _CMD_SET_PV_PRIMARY_USE = 0x10 + _CMD_SET_PV_OPTICAL_IMAGE_STABILIZATION = 0x11 + _CMD_SET_PV_HDR_VIDEO = 0x12 + _CMD_SET_PV_REGIONS_OF_INTEREST = 0x13 + _CMD_SET_INTERFACE_PRIORITY = 0x14 def __init__(self, host, port): self.host = host @@ -1933,6 +2040,35 @@ def set_flat_mode(self, mode): command = struct.pack(' 0): cv2.imshow(hl2ss.get_port_name(port), payload.image) - def display_basic(port, payload): - if (payload is not None and payload.size > 0): - cv2.imshow(hl2ss.get_port_name(port), payload) + def display_vlc(port, payload): + if (payload.image is not None and payload.image.size > 0): + cv2.imshow(hl2ss.get_port_name(port), payload.image) def display_depth_lt(port, payload): cv2.imshow(hl2ss.get_port_name(port) + '-depth', payload.depth * 8) # Scaled for visibility @@ -117,10 +125,10 @@ def display_null(port, payload): pass DISPLAY_MAP = { - hl2ss.StreamPort.RM_VLC_LEFTFRONT : display_basic, - hl2ss.StreamPort.RM_VLC_LEFTLEFT : display_basic, - hl2ss.StreamPort.RM_VLC_RIGHTFRONT : display_basic, - hl2ss.StreamPort.RM_VLC_RIGHTRIGHT : display_basic, + hl2ss.StreamPort.RM_VLC_LEFTFRONT : display_vlc, + hl2ss.StreamPort.RM_VLC_LEFTLEFT : display_vlc, + hl2ss.StreamPort.RM_VLC_RIGHTFRONT : display_vlc, + hl2ss.StreamPort.RM_VLC_RIGHTRIGHT : display_vlc, hl2ss.StreamPort.RM_DEPTH_AHAT : display_depth_ahat, hl2ss.StreamPort.RM_DEPTH_LONGTHROW : display_depth_lt, hl2ss.StreamPort.PERSONAL_VIDEO : display_pv,