OpenXR: Add interaction profiles for hand tracking

webxr/openxr/input.rs

@@ -1,9 +1,17 @@
+use std::ffi::c_void;
+use std::mem::MaybeUninit;
+
 use euclid::RigidTransform3D;
 use log::debug;
 use openxr::d3d::D3D11;
+use openxr::sys::{
+    HandJointLocationsEXT, HandJointsLocateInfoEXT, HandTrackingAimStateFB,
+    FB_HAND_TRACKING_AIM_EXTENSION_NAME,
+};
 use openxr::{
-    self, Action, ActionSet, Binding, FrameState, Hand as HandEnum, HandJoint, HandTracker,
-    Instance, Path, Posef, Session, Space, SpaceLocationFlags,
+    self, Action, ActionSet, Binding, FrameState, Hand as HandEnum, HandJoint, HandJointLocation,
+    HandTracker, HandTrackingAimFlagsFB, Instance, Path, Posef, Session, Space, SpaceLocationFlags,
+    HAND_JOINT_COUNT,
 };
 use webxr_api::Finger;
 use webxr_api::Hand;
@@ -62,19 +70,31 @@ pub struct Frame {
 }
 
 impl ClickState {
-    fn update(
+    fn update_from_action(
         &mut self,
         action: &Action<bool>,
         session: &Session<D3D11>,
         menu_selected: bool,
     ) -> (/* is_active */ bool, Option<SelectEvent>) {
         let click = action.state(session, Path::NULL).unwrap();
 
-        let select_event = if click.is_active {
-            match (click.current_state, *self) {
+        let select_event =
+            self.update_from_value(click.current_state, click.is_active, menu_selected);
+
+        (click.is_active, select_event)
+    }
+
+    fn update_from_value(
+        &mut self,
+        current_state: bool,
+        is_active: bool,
+        menu_selected: bool,
+    ) -> Option<SelectEvent> {
+        if is_active {
+            match (current_state, *self) {
                 (_, ClickState::Clicking) if menu_selected => {
                     *self = ClickState::Done;
-                    // cancel the select, we're showing a menu
+                    // Cancel the select, we're showing a menu
                     Some(SelectEvent::End)
                 }
                 (true, ClickState::Done) => {
@@ -89,12 +109,11 @@ impl ClickState {
             }
         } else if *self == ClickState::Clicking {
             *self = ClickState::Done;
-            // cancel the select, we lost tracking
+            // Cancel the select, we lost tracking
             Some(SelectEvent::End)
         } else {
             None
-        };
-        (click.is_active, select_event)
+        }
     }
 }
 
@@ -116,6 +135,7 @@ pub struct OpenXRInput {
     action_buttons_left: Vec<Action<f32>>,
     action_buttons_right: Vec<Action<f32>>,
     action_axes_common: Vec<Action<f32>>,
+    use_alternate_input_source: bool,
 }
 
 fn hand_str(h: Handedness) -> &'static str {
@@ -133,6 +153,7 @@ impl OpenXRInput {
         action_set: &ActionSet,
         session: &Session<D3D11>,
         needs_hands: bool,
+        supported_interaction_profiles: Vec<&'static str>,
     ) -> Self {
         let hand = hand_str(handedness);
         let action_aim_pose: Action<Posef> = action_set
@@ -261,6 +282,9 @@ impl OpenXRInput {
             vec![axis1, axis2, axis3, axis4]
         };
 
+        let use_alternate_input_source = supported_interaction_profiles
+            .contains(&ext_string!(FB_HAND_TRACKING_AIM_EXTENSION_NAME));
+
         Self {
             id,
             action_aim_pose,
@@ -278,6 +302,7 @@ impl OpenXRInput {
             action_axes_common,
             action_buttons_left,
             action_buttons_right,
+            use_alternate_input_source,
         }
     }
 
@@ -294,13 +319,15 @@ impl OpenXRInput {
             &action_set,
             &session,
             needs_hands,
+            supported_interaction_profiles.clone(),
         );
         let left_hand = OpenXRInput::new(
             InputId(1),
             Handedness::Left,
             &action_set,
             &session,
             needs_hands,
+            supported_interaction_profiles.clone(),
         );
 
         for profile in INTERACTION_PROFILES {
@@ -309,6 +336,11 @@ impl OpenXRInput {
                     continue;
                 }
             }
+
+            if profile.path.is_empty() {
+                continue;
+            }
+
             let select = profile.standard_buttons[0];
             let squeeze = Option::from(profile.standard_buttons[1]).filter(|&s| !s.is_empty());
             let mut bindings = right_hand.get_bindings(instance, select, squeeze, &profile);
@@ -317,6 +349,7 @@ impl OpenXRInput {
                     .get_bindings(instance, select, squeeze, &profile)
                     .into_iter(),
             );
+
             let path_controller = instance
                 .string_to_path(profile.path)
                 .expect(format!("Invalid interaction profile path: {}", profile.path).as_str());
@@ -422,7 +455,7 @@ impl OpenXRInput {
         viewer: &RigidTransform3D<f32, Viewer, Native>,
     ) -> Frame {
         use euclid::Vector3D;
-        let target_ray_origin = pose_for(&self.action_aim_space, frame_state, base_space);
+        let mut target_ray_origin = pose_for(&self.action_aim_space, frame_state, base_space);
 
         let grip_origin = pose_for(&self.action_grip_space, frame_state, base_space);
 
@@ -512,22 +545,47 @@ impl OpenXRInput {
 
         let input_changed = buttons_changed || axes_changed;
 
-        let (click_is_active, click_event) =
+        let (click_is_active, mut click_event) = if !self.use_alternate_input_source {
             self.click_state
-                .update(&self.action_click, session, menu_selected);
+                .update_from_action(&self.action_click, session, menu_selected)
+        } else {
+            (true, None)
+        };
         let (squeeze_is_active, squeeze_event) =
             self.squeeze_state
-                .update(&self.action_squeeze, session, menu_selected);
+                .update_from_action(&self.action_squeeze, session, menu_selected);
+
+        let mut aim_state: Option<HandTrackingAimStateFB> = None;
+        let hand = self.hand_tracker.as_ref().and_then(|tracker| {
+            locate_hand(
+                base_space,
+                tracker,
+                frame_state,
+                self.use_alternate_input_source,
+                session,
+                &mut aim_state,
+            )
+        });
 
-        let hand = target_ray_origin
-            .and_then(|_origin| self.hand_tracker.as_ref())
-            .and_then(|tracker| locate_hand(base_space, tracker, frame_state));
+        let mut pressed = click_is_active && click.current_state;
+        let squeezed = squeeze_is_active && squeeze.current_state;
+
+        if let Some(state) = aim_state {
+            target_ray_origin.replace(super::transform(&state.aim_pose));
+            let index_pinching = state
+                .status
+                .intersects(HandTrackingAimFlagsFB::INDEX_PINCHING);
+            click_event = self
+                .click_state
+                .update_from_value(index_pinching, true, menu_selected);
+            pressed = index_pinching;
+        }
 
         let input_frame = InputFrame {
             target_ray_origin,
             id: self.id,
-            pressed: click_is_active && click.current_state,
-            squeezed: squeeze_is_active && squeeze.current_state,
+            pressed,
+            squeezed,
             grip_origin,
             hand,
             button_values,
@@ -583,8 +641,51 @@ fn locate_hand(
     base_space: &Space,
     tracker: &HandTracker,
     frame_state: &FrameState,
+    use_alternate_input_source: bool,
+    session: &Session<D3D11>,
+    aim_state: &mut Option<HandTrackingAimStateFB>,
 ) -> Option<Box<Hand<JointFrame>>> {
-    let locations = base_space.locate_hand_joints(tracker, frame_state.predicted_display_time);
+    let mut state = HandTrackingAimStateFB::out(std::ptr::null_mut());
+    let locations = {
+        if !use_alternate_input_source {
+            base_space.locate_hand_joints(tracker, frame_state.predicted_display_time)
+        } else {
+            let locate_info = HandJointsLocateInfoEXT {
+                ty: HandJointsLocateInfoEXT::TYPE,
+                next: std::ptr::null(),
+                base_space: base_space.as_raw(),
+                time: frame_state.predicted_display_time,
+            };
+
+            let mut locations = MaybeUninit::<[HandJointLocation; HAND_JOINT_COUNT]>::uninit();
+            let mut location_info = HandJointLocationsEXT {
+                ty: HandJointLocationsEXT::TYPE,
+                next: &mut state as *mut _ as *mut c_void,
+                is_active: false.into(),
+                joint_count: HAND_JOINT_COUNT as u32,
+                joint_locations: locations.as_mut_ptr() as _,
+            };
+
+            // Check if hand tracking is supported by the session instance
+            let raw_hand_tracker = session.instance().exts().ext_hand_tracking.as_ref()?;
+
+            unsafe {
+                Ok(
+                    match (raw_hand_tracker.locate_hand_joints)(
+                        tracker.as_raw(),
+                        &locate_info,
+                        &mut location_info,
+                    ) {
+                        openxr::sys::Result::SUCCESS if location_info.is_active.into() => {
+                            aim_state.replace(state.assume_init());
+                            Some(locations.assume_init())
+                        }
+                        _ => None,
+                    },
+                )
+            }
+        }
+    };
     let locations = if let Ok(Some(ref locations)) = locations {
         Hand {
             wrist: Some(&locations[HandJoint::WRIST]),

webxr/openxr/interaction_profiles.rs

@@ -1,7 +1,9 @@
 use openxr::{
     sys::{
-        BD_CONTROLLER_INTERACTION_EXTENSION_NAME, EXT_HP_MIXED_REALITY_CONTROLLER_EXTENSION_NAME,
-        EXT_SAMSUNG_ODYSSEY_CONTROLLER_EXTENSION_NAME, FB_TOUCH_CONTROLLER_PRO_EXTENSION_NAME,
+        BD_CONTROLLER_INTERACTION_EXTENSION_NAME, EXT_HAND_INTERACTION_EXTENSION_NAME,
+        EXT_HP_MIXED_REALITY_CONTROLLER_EXTENSION_NAME,
+        EXT_SAMSUNG_ODYSSEY_CONTROLLER_EXTENSION_NAME, FB_HAND_TRACKING_AIM_EXTENSION_NAME,
+        FB_TOUCH_CONTROLLER_PRO_EXTENSION_NAME,
         HTC_VIVE_COSMOS_CONTROLLER_INTERACTION_EXTENSION_NAME,
         HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME,
         META_TOUCH_CONTROLLER_PLUS_EXTENSION_NAME, ML_ML2_CONTROLLER_INTERACTION_EXTENSION_NAME,
@@ -16,7 +18,7 @@ macro_rules! ext_string {
     };
 }
 
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub enum InteractionProfileType {
     KhrSimpleController,
     BytedancePicoNeo3Controller,
@@ -38,10 +40,11 @@ pub enum InteractionProfileType {
     MetaTouchControllerQuest2,
     SamsungOdysseyController,
     ValveIndexController,
+    ExtHandInteraction,
+    FbHandTrackingAim,
 }
 
 #[derive(Clone, Copy, Debug)]
-#[allow(unused)]
 pub struct InteractionProfile<'a> {
     pub profile_type: InteractionProfileType,
     /// The interaction profile path
@@ -334,7 +337,29 @@ pub static VALVE_INDEX_CONTROLLER_PROFILE: InteractionProfile = InteractionProfi
     profiles: &["valve-index", "generic-trigger-squeeze-touchpad-thumbstick"],
 };
 
-pub static INTERACTION_PROFILES: [InteractionProfile; 20] = [
+pub static EXT_HAND_INTERACTION_PROFILE: InteractionProfile = InteractionProfile {
+    profile_type: InteractionProfileType::ExtHandInteraction,
+    path: "interaction_profiles/ext/hand_interaction_ext",
+    required_extension: Some(EXT_HAND_INTERACTION_EXTENSION_NAME),
+    standard_buttons: &["pinch_ext/value", "", "", ""],
+    standard_axes: &["", "", "", ""],
+    left_buttons: &[],
+    right_buttons: &[],
+    profiles: &["generic-hand-select", "generic-hand"],
+};
+
+pub static FB_HAND_TRACKING_AIM_PROFILE: InteractionProfile = InteractionProfile {
+    profile_type: InteractionProfileType::FbHandTrackingAim,
+    path: "",
+    required_extension: Some(FB_HAND_TRACKING_AIM_EXTENSION_NAME),
+    standard_buttons: &["", "", "", ""],
+    standard_axes: &["", "", "", ""],
+    left_buttons: &[],
+    right_buttons: &[],
+    profiles: &["generic-hand-select", "generic-hand"],
+};
+
+pub static INTERACTION_PROFILES: [InteractionProfile; 22] = [
     KHR_SIMPLE_CONTROLLER_PROFILE,
     BYTEDANCE_PICO_NEO3_CONTROLLER_PROFILE,
     BYTEDANCE_PICO_4_CONTROLLER_PROFILE,
@@ -355,6 +380,8 @@ pub static INTERACTION_PROFILES: [InteractionProfile; 20] = [
     META_TOUCH_CONTROLLER_QUEST_2_PROFILE,
     SAMSUNG_ODYSSEY_CONTROLLER_PROFILE,
     VALVE_INDEX_CONTROLLER_PROFILE,
+    EXT_HAND_INTERACTION_PROFILE,
+    FB_HAND_TRACKING_AIM_PROFILE,
 ];
 
 pub fn get_profiles_from_path(path: String) -> &'static [&'static str] {
@@ -405,5 +432,13 @@ pub fn get_supported_interaction_profiles(
         extensions.push(ext_string!(META_TOUCH_CONTROLLER_PLUS_EXTENSION_NAME));
         enabled_extensions.meta_touch_controller_plus = true;
     }
+    if supported_extensions.ext_hand_interaction {
+        extensions.push(ext_string!(EXT_HAND_INTERACTION_EXTENSION_NAME));
+        enabled_extensions.ext_hand_interaction = true;
+    }
+    if supported_extensions.fb_hand_tracking_aim {
+        extensions.push(ext_string!(FB_HAND_TRACKING_AIM_EXTENSION_NAME));
+        enabled_extensions.fb_hand_tracking_aim = true;
+    }
     extensions
 }