Skip to content
This repository has been archived by the owner on Feb 16, 2025. It is now read-only.

Commit

Permalink
OpenXR: Add interaction profiles for hand tracking (#236)
Browse files Browse the repository at this point in the history 
* Get hand info from HandTrackingAimStateFB

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* Un-tuple locate_hand, set aimstate via ref instead

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* Add profile for XR_EXT_hand_interaction

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* Use update_from_value in update_from_action

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* Add use_alternate_input_source bool on OpenXRInput

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* Adjust hand jointframe assignment

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* formatting

Signed-off-by: Daniel Adams <msub2official@gmail.com>

* Remove unused import

Signed-off-by: Daniel Adams <msub2official@gmail.com>

---------

Signed-off-by: Daniel Adams <msub2official@gmail.com>
  • Loading branch information
@msub2
msub2 authored Aug 12, 2024
1 parent 355dce2 commit dd76329
Show file tree
Hide file tree
Showing 2 changed files with 160 additions and 24 deletions.
139 changes: 120 additions & 19 deletions webxr/openxr/input.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
use std::ffi::c_void;
use std::mem::MaybeUninit;

use euclid::RigidTransform3D;
use log::debug;
use openxr::d3d::D3D11;
use openxr::sys::{
HandJointLocationsEXT, HandJointsLocateInfoEXT, HandTrackingAimStateFB,
FB_HAND_TRACKING_AIM_EXTENSION_NAME,
};
use openxr::{
self, Action, ActionSet, Binding, FrameState, Hand as HandEnum, HandJoint, HandTracker,
Instance, Path, Posef, Session, Space, SpaceLocationFlags,
self, Action, ActionSet, Binding, FrameState, Hand as HandEnum, HandJoint, HandJointLocation,
HandTracker, HandTrackingAimFlagsFB, Instance, Path, Posef, Session, Space, SpaceLocationFlags,
HAND_JOINT_COUNT,
};
use webxr_api::Finger;
use webxr_api::Hand;
Expand Down Expand Up @@ -62,19 +70,31 @@ pub struct Frame {
}

impl ClickState {
fn update(
fn update_from_action(
&mut self,
action: &Action<bool>,
session: &Session<D3D11>,
menu_selected: bool,
) -> (/* is_active */ bool, Option<SelectEvent>) {
let click = action.state(session, Path::NULL).unwrap();

let select_event = if click.is_active {
match (click.current_state, *self) {
let select_event =
self.update_from_value(click.current_state, click.is_active, menu_selected);

(click.is_active, select_event)
}

fn update_from_value(
&mut self,
current_state: bool,
is_active: bool,
menu_selected: bool,
) -> Option<SelectEvent> {
if is_active {
match (current_state, *self) {
(_, ClickState::Clicking) if menu_selected => {
*self = ClickState::Done;
// cancel the select, we're showing a menu
// Cancel the select, we're showing a menu
Some(SelectEvent::End)
}
(true, ClickState::Done) => {
Expand All @@ -89,12 +109,11 @@ impl ClickState {
}
} else if *self == ClickState::Clicking {
*self = ClickState::Done;
// cancel the select, we lost tracking
// Cancel the select, we lost tracking
Some(SelectEvent::End)
} else {
None
};
(click.is_active, select_event)
}
}
}

Expand All @@ -116,6 +135,7 @@ pub struct OpenXRInput {
action_buttons_left: Vec<Action<f32>>,
action_buttons_right: Vec<Action<f32>>,
action_axes_common: Vec<Action<f32>>,
use_alternate_input_source: bool,
}

fn hand_str(h: Handedness) -> &'static str {
Expand All @@ -133,6 +153,7 @@ impl OpenXRInput {
action_set: &ActionSet,
session: &Session<D3D11>,
needs_hands: bool,
supported_interaction_profiles: Vec<&'static str>,
) -> Self {
let hand = hand_str(handedness);
let action_aim_pose: Action<Posef> = action_set
Expand Down Expand Up @@ -261,6 +282,9 @@ impl OpenXRInput {
vec![axis1, axis2, axis3, axis4]
};

let use_alternate_input_source = supported_interaction_profiles
.contains(&ext_string!(FB_HAND_TRACKING_AIM_EXTENSION_NAME));

Self {
id,
action_aim_pose,
Expand All @@ -278,6 +302,7 @@ impl OpenXRInput {
action_axes_common,
action_buttons_left,
action_buttons_right,
use_alternate_input_source,
}
}

Expand All @@ -294,13 +319,15 @@ impl OpenXRInput {
&action_set,
&session,
needs_hands,
supported_interaction_profiles.clone(),
);
let left_hand = OpenXRInput::new(
InputId(1),
Handedness::Left,
&action_set,
&session,
needs_hands,
supported_interaction_profiles.clone(),
);

for profile in INTERACTION_PROFILES {
Expand All @@ -309,6 +336,11 @@ impl OpenXRInput {
continue;
}
}

if profile.path.is_empty() {
continue;
}

let select = profile.standard_buttons[0];
let squeeze = Option::from(profile.standard_buttons[1]).filter(|&s| !s.is_empty());
let mut bindings = right_hand.get_bindings(instance, select, squeeze, &profile);
Expand All @@ -317,6 +349,7 @@ impl OpenXRInput {
.get_bindings(instance, select, squeeze, &profile)
.into_iter(),
);

let path_controller = instance
.string_to_path(profile.path)
.expect(format!("Invalid interaction profile path: {}", profile.path).as_str());
Expand Down Expand Up @@ -422,7 +455,7 @@ impl OpenXRInput {
viewer: &RigidTransform3D<f32, Viewer, Native>,
) -> Frame {
use euclid::Vector3D;
let target_ray_origin = pose_for(&self.action_aim_space, frame_state, base_space);
let mut target_ray_origin = pose_for(&self.action_aim_space, frame_state, base_space);

let grip_origin = pose_for(&self.action_grip_space, frame_state, base_space);

Expand Down Expand Up @@ -512,22 +545,47 @@ impl OpenXRInput {

let input_changed = buttons_changed || axes_changed;

let (click_is_active, click_event) =
let (click_is_active, mut click_event) = if !self.use_alternate_input_source {
self.click_state
.update(&self.action_click, session, menu_selected);
.update_from_action(&self.action_click, session, menu_selected)
} else {
(true, None)
};
let (squeeze_is_active, squeeze_event) =
self.squeeze_state
.update(&self.action_squeeze, session, menu_selected);
.update_from_action(&self.action_squeeze, session, menu_selected);

let mut aim_state: Option<HandTrackingAimStateFB> = None;
let hand = self.hand_tracker.as_ref().and_then(|tracker| {
locate_hand(
base_space,
tracker,
frame_state,
self.use_alternate_input_source,
session,
&mut aim_state,
)
});

let hand = target_ray_origin
.and_then(|_origin| self.hand_tracker.as_ref())
.and_then(|tracker| locate_hand(base_space, tracker, frame_state));
let mut pressed = click_is_active && click.current_state;
let squeezed = squeeze_is_active && squeeze.current_state;

if let Some(state) = aim_state {
target_ray_origin.replace(super::transform(&state.aim_pose));
let index_pinching = state
.status
.intersects(HandTrackingAimFlagsFB::INDEX_PINCHING);
click_event = self
.click_state
.update_from_value(index_pinching, true, menu_selected);
pressed = index_pinching;
}

let input_frame = InputFrame {
target_ray_origin,
id: self.id,
pressed: click_is_active && click.current_state,
squeezed: squeeze_is_active && squeeze.current_state,
pressed,
squeezed,
grip_origin,
hand,
button_values,
Expand Down Expand Up @@ -583,8 +641,51 @@ fn locate_hand(
base_space: &Space,
tracker: &HandTracker,
frame_state: &FrameState,
use_alternate_input_source: bool,
session: &Session<D3D11>,
aim_state: &mut Option<HandTrackingAimStateFB>,
) -> Option<Box<Hand<JointFrame>>> {
let locations = base_space.locate_hand_joints(tracker, frame_state.predicted_display_time);
let mut state = HandTrackingAimStateFB::out(std::ptr::null_mut());
let locations = {
if !use_alternate_input_source {
base_space.locate_hand_joints(tracker, frame_state.predicted_display_time)
} else {
let locate_info = HandJointsLocateInfoEXT {
ty: HandJointsLocateInfoEXT::TYPE,
next: std::ptr::null(),
base_space: base_space.as_raw(),
time: frame_state.predicted_display_time,
};

let mut locations = MaybeUninit::<[HandJointLocation; HAND_JOINT_COUNT]>::uninit();
let mut location_info = HandJointLocationsEXT {
ty: HandJointLocationsEXT::TYPE,
next: &mut state as *mut _ as *mut c_void,
is_active: false.into(),
joint_count: HAND_JOINT_COUNT as u32,
joint_locations: locations.as_mut_ptr() as _,
};

// Check if hand tracking is supported by the session instance
let raw_hand_tracker = session.instance().exts().ext_hand_tracking.as_ref()?;

unsafe {
Ok(
match (raw_hand_tracker.locate_hand_joints)(
tracker.as_raw(),
&locate_info,
&mut location_info,
) {
openxr::sys::Result::SUCCESS if location_info.is_active.into() => {
aim_state.replace(state.assume_init());
Some(locations.assume_init())
}
_ => None,
},
)
}
}
};
let locations = if let Ok(Some(ref locations)) = locations {
Hand {
wrist: Some(&locations[HandJoint::WRIST]),
Expand Down
45 changes: 40 additions & 5 deletions webxr/openxr/interaction_profiles.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
use openxr::{
sys::{
BD_CONTROLLER_INTERACTION_EXTENSION_NAME, EXT_HP_MIXED_REALITY_CONTROLLER_EXTENSION_NAME,
EXT_SAMSUNG_ODYSSEY_CONTROLLER_EXTENSION_NAME, FB_TOUCH_CONTROLLER_PRO_EXTENSION_NAME,
BD_CONTROLLER_INTERACTION_EXTENSION_NAME, EXT_HAND_INTERACTION_EXTENSION_NAME,
EXT_HP_MIXED_REALITY_CONTROLLER_EXTENSION_NAME,
EXT_SAMSUNG_ODYSSEY_CONTROLLER_EXTENSION_NAME, FB_HAND_TRACKING_AIM_EXTENSION_NAME,
FB_TOUCH_CONTROLLER_PRO_EXTENSION_NAME,
HTC_VIVE_COSMOS_CONTROLLER_INTERACTION_EXTENSION_NAME,
HTC_VIVE_FOCUS3_CONTROLLER_INTERACTION_EXTENSION_NAME,
META_TOUCH_CONTROLLER_PLUS_EXTENSION_NAME, ML_ML2_CONTROLLER_INTERACTION_EXTENSION_NAME,
Expand All @@ -16,7 +18,7 @@ macro_rules! ext_string {
};
}

#[derive(Clone, Copy, Debug)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum InteractionProfileType {
KhrSimpleController,
BytedancePicoNeo3Controller,
Expand All @@ -38,10 +40,11 @@ pub enum InteractionProfileType {
MetaTouchControllerQuest2,
SamsungOdysseyController,
ValveIndexController,
ExtHandInteraction,
FbHandTrackingAim,
}

#[derive(Clone, Copy, Debug)]
#[allow(unused)]
pub struct InteractionProfile<'a> {
pub profile_type: InteractionProfileType,
/// The interaction profile path
Expand Down Expand Up @@ -334,7 +337,29 @@ pub static VALVE_INDEX_CONTROLLER_PROFILE: InteractionProfile = InteractionProfi
profiles: &["valve-index", "generic-trigger-squeeze-touchpad-thumbstick"],
};

pub static INTERACTION_PROFILES: [InteractionProfile; 20] = [
pub static EXT_HAND_INTERACTION_PROFILE: InteractionProfile = InteractionProfile {
profile_type: InteractionProfileType::ExtHandInteraction,
path: "interaction_profiles/ext/hand_interaction_ext",
required_extension: Some(EXT_HAND_INTERACTION_EXTENSION_NAME),
standard_buttons: &["pinch_ext/value", "", "", ""],
standard_axes: &["", "", "", ""],
left_buttons: &[],
right_buttons: &[],
profiles: &["generic-hand-select", "generic-hand"],
};

pub static FB_HAND_TRACKING_AIM_PROFILE: InteractionProfile = InteractionProfile {
profile_type: InteractionProfileType::FbHandTrackingAim,
path: "",
required_extension: Some(FB_HAND_TRACKING_AIM_EXTENSION_NAME),
standard_buttons: &["", "", "", ""],
standard_axes: &["", "", "", ""],
left_buttons: &[],
right_buttons: &[],
profiles: &["generic-hand-select", "generic-hand"],
};

pub static INTERACTION_PROFILES: [InteractionProfile; 22] = [
KHR_SIMPLE_CONTROLLER_PROFILE,
BYTEDANCE_PICO_NEO3_CONTROLLER_PROFILE,
BYTEDANCE_PICO_4_CONTROLLER_PROFILE,
Expand All @@ -355,6 +380,8 @@ pub static INTERACTION_PROFILES: [InteractionProfile; 20] = [
META_TOUCH_CONTROLLER_QUEST_2_PROFILE,
SAMSUNG_ODYSSEY_CONTROLLER_PROFILE,
VALVE_INDEX_CONTROLLER_PROFILE,
EXT_HAND_INTERACTION_PROFILE,
FB_HAND_TRACKING_AIM_PROFILE,
];

pub fn get_profiles_from_path(path: String) -> &'static [&'static str] {
Expand Down Expand Up @@ -405,5 +432,13 @@ pub fn get_supported_interaction_profiles(
extensions.push(ext_string!(META_TOUCH_CONTROLLER_PLUS_EXTENSION_NAME));
enabled_extensions.meta_touch_controller_plus = true;
}
if supported_extensions.ext_hand_interaction {
extensions.push(ext_string!(EXT_HAND_INTERACTION_EXTENSION_NAME));
enabled_extensions.ext_hand_interaction = true;
}
if supported_extensions.fb_hand_tracking_aim {
extensions.push(ext_string!(FB_HAND_TRACKING_AIM_EXTENSION_NAME));
enabled_extensions.fb_hand_tracking_aim = true;
}
extensions
}

0 comments on commit dd76329

Please sign in to comment.