Hand tracking gestures (#1794)
* add settings for hand tracking * hands_to_gestures to get gestures from hand data * activate button presses for gestures * adjust some values to improve gesture accuracy * move gesture binds into hands_to_gestures * remember to set empty binds on dummy list * fix build issues * remove unnecessary enumerate * support curl and add grip gesture * make finger thresholds slightly larger * add curl distance settings * curl: use min of proximal and metacarpal * add joystick press gesture + correct curl maths * improve help strings for curl and pinch values * remove warns for curling * fix broken right hand binds * remove button press workaround * support vertical joystick control using thumb curl all fingers and touch thumb to index finger. centre is between tip and intermediate bone * disable index pinching when controlling the joystick * 0.0 not 0 :c * dual axis joystick support somewhat janky, likely just an issue with specific offsets and numbers * add gesture joystick deadzone * config joystick deadzone and offset * fix clippy errors * format * initial refactor work is_gesture_active * migration to handgesturemanager (untested) * make gesture activation work * touch binds + correct click binds * increase default stop delay * rename gesture delay settings * fix joystick click * fmt * I DID IT !!!! JOYSTICK !!!!!!! * rename hover to value * remove hand tracking toggle * handed abxy IDs * format * use button mapping manager (slightly broken?) * format * review response :) * fmt * add joystick range setting + increase default deadzone * review response sequel * fmt * fix collapsible on hand tracking settings * code smell :c
This commit is contained in:
Barnaby
GitHub
parent
a3ce7c1364
commit
ae66ce92c9
|
|
@ -1,6 +1,7 @@
|
|||
use crate::{
|
||||
bitrate::BitrateManager,
|
||||
face_tracking::FaceTrackingSink,
|
||||
hand_gestures::{trigger_hand_gesture_actions, HandGestureManager, HAND_GESTURE_BUTTON_SET},
|
||||
haptics,
|
||||
input_mapping::ButtonMappingManager,
|
||||
sockets::WelcomeSocket,
|
||||
|
|
@ -643,9 +644,27 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> ConResult {
|
|||
};
|
||||
|
||||
let tracking_manager = Arc::new(Mutex::new(TrackingManager::new()));
|
||||
let hand_gesture_manager = Arc::new(Mutex::new(HandGestureManager::new()));
|
||||
|
||||
let tracking_receive_thread = thread::spawn({
|
||||
let tracking_manager = Arc::clone(&tracking_manager);
|
||||
let hand_gesture_manager = Arc::clone(&hand_gesture_manager);
|
||||
|
||||
let mut gestures_button_mapping_manager = SERVER_DATA_MANAGER
|
||||
.read()
|
||||
.settings()
|
||||
.headset
|
||||
.controllers
|
||||
.as_option()
|
||||
.and_then(|config| {
|
||||
config.hand_tracking.use_gestures.as_option().and_then(|_| {
|
||||
Some(ButtonMappingManager::new_automatic(
|
||||
&HAND_GESTURE_BUTTON_SET,
|
||||
&config.button_mapping_config,
|
||||
))
|
||||
})
|
||||
});
|
||||
|
||||
move || {
|
||||
let mut face_tracking_sink =
|
||||
settings
|
||||
|
|
@ -657,7 +676,7 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> ConResult {
|
|||
});
|
||||
|
||||
let mut track_controllers = 0u32;
|
||||
if let Switch::Enabled(config) = settings.headset.controllers {
|
||||
if let Switch::Enabled(config) = &settings.headset.controllers {
|
||||
track_controllers = config.tracked.into();
|
||||
}
|
||||
|
||||
|
|
@ -679,6 +698,7 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> ConResult {
|
|||
{
|
||||
let data_manager_lock = SERVER_DATA_MANAGER.read();
|
||||
let config = &data_manager_lock.settings().headset;
|
||||
|
||||
motions = tracking_manager_lock.transform_motions(
|
||||
config,
|
||||
&tracking.device_motions,
|
||||
|
|
@ -740,14 +760,57 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> ConResult {
|
|||
.into_iter()
|
||||
.map(|(id, motion)| tracking::to_ffi_motion(id, motion))
|
||||
.collect::<Vec<_>>();
|
||||
let ffi_left_hand_skeleton = left_hand_skeleton.map(tracking::to_ffi_skeleton);
|
||||
let ffi_right_hand_skeleton = right_hand_skeleton.map(tracking::to_ffi_skeleton);
|
||||
let mut ffi_left_hand_skeleton = left_hand_skeleton.map(tracking::to_ffi_skeleton);
|
||||
let mut ffi_right_hand_skeleton =
|
||||
right_hand_skeleton.map(tracking::to_ffi_skeleton);
|
||||
|
||||
drop(tracking_manager_lock);
|
||||
|
||||
if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
|
||||
stats.report_tracking_received(tracking.target_timestamp);
|
||||
|
||||
// Handle hand gestures
|
||||
if let Switch::Enabled(controllers_config) = &settings.headset.controllers {
|
||||
if let Switch::Enabled(gestures_config) =
|
||||
&controllers_config.hand_tracking.use_gestures
|
||||
{
|
||||
let mut hand_gesture_manager_lock = hand_gesture_manager.lock();
|
||||
|
||||
if let Some(hand_skeleton) = tracking.hand_skeletons[0] {
|
||||
trigger_hand_gesture_actions(
|
||||
gestures_button_mapping_manager.as_mut().unwrap(),
|
||||
*LEFT_HAND_ID,
|
||||
&hand_gesture_manager_lock.get_active_gestures(
|
||||
hand_skeleton,
|
||||
gestures_config.clone(),
|
||||
*LEFT_HAND_ID,
|
||||
),
|
||||
);
|
||||
}
|
||||
if let Some(hand_skeleton) = tracking.hand_skeletons[1] {
|
||||
trigger_hand_gesture_actions(
|
||||
gestures_button_mapping_manager.as_mut().unwrap(),
|
||||
*RIGHT_HAND_ID,
|
||||
&hand_gesture_manager_lock.get_active_gestures(
|
||||
hand_skeleton,
|
||||
gestures_config.clone(),
|
||||
*RIGHT_HAND_ID,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let data_manager_lock = SERVER_DATA_MANAGER.read();
|
||||
let config = &data_manager_lock.settings().headset;
|
||||
|
||||
if let Switch::Enabled(controllers) = &config.controllers {
|
||||
if !controllers.hand_tracking.enable_skeleton {
|
||||
ffi_left_hand_skeleton = None;
|
||||
ffi_right_hand_skeleton = None;
|
||||
}
|
||||
}
|
||||
|
||||
unsafe {
|
||||
crate::SetTracking(
|
||||
tracking.target_timestamp.as_nanos() as _,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,599 @@
|
|||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::Hash,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use alvr_common::{
|
||||
glam::{Vec2, Vec3},
|
||||
once_cell::sync::Lazy,
|
||||
*,
|
||||
};
|
||||
|
||||
use alvr_packets::ButtonValue;
|
||||
use alvr_session::HandGestureConfig;
|
||||
|
||||
use crate::input_mapping::ButtonMappingManager;
|
||||
|
||||
fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose {
|
||||
Pose {
|
||||
orientation: a.orientation.lerp(b.orientation, fac),
|
||||
position: a.position.lerp(b.position, fac),
|
||||
}
|
||||
}
|
||||
|
||||
pub static HAND_GESTURE_BUTTON_SET: Lazy<HashSet<u64>> = Lazy::new(|| {
|
||||
[
|
||||
*LEFT_X_CLICK_ID,
|
||||
*LEFT_X_TOUCH_ID,
|
||||
*LEFT_Y_CLICK_ID,
|
||||
*LEFT_Y_TOUCH_ID,
|
||||
*LEFT_MENU_CLICK_ID,
|
||||
*LEFT_SQUEEZE_VALUE_ID,
|
||||
*LEFT_TRIGGER_VALUE_ID,
|
||||
*LEFT_TRIGGER_TOUCH_ID,
|
||||
*LEFT_THUMBSTICK_X_ID,
|
||||
*LEFT_THUMBSTICK_Y_ID,
|
||||
*LEFT_THUMBSTICK_CLICK_ID,
|
||||
*LEFT_THUMBSTICK_TOUCH_ID,
|
||||
*RIGHT_A_CLICK_ID,
|
||||
*RIGHT_A_TOUCH_ID,
|
||||
*RIGHT_B_CLICK_ID,
|
||||
*RIGHT_B_TOUCH_ID,
|
||||
*RIGHT_SQUEEZE_VALUE_ID,
|
||||
*RIGHT_TRIGGER_VALUE_ID,
|
||||
*RIGHT_TRIGGER_TOUCH_ID,
|
||||
*RIGHT_THUMBSTICK_X_ID,
|
||||
*RIGHT_THUMBSTICK_Y_ID,
|
||||
*RIGHT_THUMBSTICK_CLICK_ID,
|
||||
*RIGHT_THUMBSTICK_TOUCH_ID,
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
});
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HandGesture {
|
||||
pub id: HandGestureId,
|
||||
pub active: bool,
|
||||
pub clicked: bool,
|
||||
pub touching: bool,
|
||||
pub value: f32,
|
||||
}
|
||||
|
||||
pub struct GestureAction {
|
||||
last_activated: u128,
|
||||
last_deactivated: u128,
|
||||
entering: bool,
|
||||
entering_since: u128,
|
||||
exiting: bool,
|
||||
exiting_since: u128,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq)]
|
||||
#[allow(dead_code)]
|
||||
pub enum HandGestureId {
|
||||
// Pinches
|
||||
ThumbIndexPinch,
|
||||
ThumbMiddlePinch,
|
||||
ThumbRingPinch,
|
||||
ThumbLittlePinch,
|
||||
// Curls
|
||||
ThumbCurl,
|
||||
IndexCurl,
|
||||
MiddleCurl,
|
||||
RingCurl,
|
||||
LittleCurl,
|
||||
GripCurl,
|
||||
// Complex
|
||||
JoystickX,
|
||||
JoystickY,
|
||||
}
|
||||
|
||||
pub struct HandGestureManager {
|
||||
gesture_data_left: HashMap<HandGestureId, GestureAction>,
|
||||
gesture_data_right: HashMap<HandGestureId, GestureAction>,
|
||||
}
|
||||
|
||||
impl HandGestureManager {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
gesture_data_left: HashMap::new(),
|
||||
gesture_data_right: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_active_gestures(
|
||||
&mut self,
|
||||
hand_skeleton: [Pose; 26],
|
||||
config: HandGestureConfig,
|
||||
device_id: u64,
|
||||
) -> Vec<HandGesture> {
|
||||
// global joints
|
||||
let gj = hand_skeleton;
|
||||
|
||||
// if we model the tip of the finger as a spherical object, we should account for its radius
|
||||
// these are intentionally under the average by ~5mm since the touch and trigger distances are already configurable in settings
|
||||
let thumb_rad: f32 = 0.0075; // average thumb is ~20mm in diameter
|
||||
let index_rad: f32 = 0.0065; // average index finger is ~18mm in diameter
|
||||
let middle_rad: f32 = 0.0065; // average middle finger is ~18mm in diameter
|
||||
let ring_rad: f32 = 0.006; // average ring finger is ~17mm in diameter
|
||||
let little_rad: f32 = 0.005; // average pinky finger is ~15mm in diameter
|
||||
let palm_depth: f32 = 0.005; // average palm bones are ~10mm from the skin
|
||||
|
||||
// we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers
|
||||
let pinch_min = config.pinch_touch_distance * 0.01;
|
||||
let pinch_max = config.pinch_trigger_distance * 0.01;
|
||||
let curl_min = config.curl_touch_distance * 0.01;
|
||||
let curl_max = config.curl_trigger_distance * 0.01;
|
||||
|
||||
let palm: Pose = gj[0];
|
||||
let thumb_tip: Pose = gj[5];
|
||||
let index_metacarpal: Pose = gj[6];
|
||||
let index_proximal: Pose = gj[7];
|
||||
let index_intermediate: Pose = gj[8];
|
||||
let index_distal: Pose = gj[9];
|
||||
let index_tip: Pose = gj[10];
|
||||
let middle_metacarpal: Pose = gj[11];
|
||||
let middle_proximal: Pose = gj[12];
|
||||
let middle_tip: Pose = gj[15];
|
||||
let ring_metacarpal: Pose = gj[16];
|
||||
let ring_proximal: Pose = gj[17];
|
||||
let ring_tip: Pose = gj[20];
|
||||
let little_metacarpal: Pose = gj[21];
|
||||
let little_proximal: Pose = gj[22];
|
||||
let little_tip: Pose = gj[25];
|
||||
|
||||
let mut gestures: Vec<HandGesture> = Vec::new();
|
||||
|
||||
// Thumb & index pinch
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::ThumbIndexPinch,
|
||||
active: self.is_gesture_active(
|
||||
HandGestureId::ThumbIndexPinch,
|
||||
thumb_tip,
|
||||
thumb_rad,
|
||||
index_tip,
|
||||
index_rad,
|
||||
pinch_max,
|
||||
config.repeat_delay,
|
||||
config.activation_delay,
|
||||
config.deactivation_delay,
|
||||
device_id,
|
||||
),
|
||||
clicked: self.test_gesture_dist(thumb_tip, thumb_rad, index_tip, index_rad, pinch_min),
|
||||
touching: self.test_gesture_dist(thumb_tip, thumb_rad, index_tip, index_rad, pinch_max),
|
||||
value: self.get_gesture_hover(
|
||||
thumb_tip, thumb_rad, index_tip, index_rad, pinch_min, pinch_max,
|
||||
),
|
||||
});
|
||||
|
||||
// Thumb & middle pinch
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::ThumbMiddlePinch,
|
||||
active: self.is_gesture_active(
|
||||
HandGestureId::ThumbMiddlePinch,
|
||||
thumb_tip,
|
||||
thumb_rad,
|
||||
middle_tip,
|
||||
middle_rad,
|
||||
pinch_max,
|
||||
config.repeat_delay,
|
||||
config.activation_delay,
|
||||
config.deactivation_delay,
|
||||
device_id,
|
||||
),
|
||||
clicked: self
|
||||
.test_gesture_dist(thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min),
|
||||
touching: self
|
||||
.test_gesture_dist(thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_max),
|
||||
value: self.get_gesture_hover(
|
||||
thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min, pinch_max,
|
||||
),
|
||||
});
|
||||
|
||||
// Thumb & ring pinch
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::ThumbRingPinch,
|
||||
active: self.is_gesture_active(
|
||||
HandGestureId::ThumbRingPinch,
|
||||
thumb_tip,
|
||||
thumb_rad,
|
||||
ring_tip,
|
||||
ring_rad,
|
||||
pinch_max,
|
||||
config.repeat_delay,
|
||||
config.activation_delay,
|
||||
config.deactivation_delay,
|
||||
device_id,
|
||||
),
|
||||
clicked: self.test_gesture_dist(thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min),
|
||||
touching: self.test_gesture_dist(thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_max),
|
||||
value: self.get_gesture_hover(
|
||||
thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min, pinch_max,
|
||||
),
|
||||
});
|
||||
|
||||
// Thumb & little pinch
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::ThumbLittlePinch,
|
||||
active: self.is_gesture_active(
|
||||
HandGestureId::ThumbLittlePinch,
|
||||
thumb_tip,
|
||||
thumb_rad,
|
||||
little_tip,
|
||||
little_rad,
|
||||
pinch_max,
|
||||
config.repeat_delay,
|
||||
config.activation_delay,
|
||||
config.deactivation_delay,
|
||||
device_id,
|
||||
),
|
||||
clicked: self
|
||||
.test_gesture_dist(thumb_tip, thumb_rad, little_tip, little_rad, pinch_min),
|
||||
touching: self
|
||||
.test_gesture_dist(thumb_tip, thumb_rad, little_tip, little_rad, pinch_max),
|
||||
value: self.get_gesture_hover(
|
||||
thumb_tip, thumb_rad, little_tip, little_rad, pinch_min, pinch_max,
|
||||
),
|
||||
});
|
||||
|
||||
// Finger curls
|
||||
let thumb_curl =
|
||||
self.get_gesture_hover(palm, palm_depth, thumb_tip, thumb_rad, curl_min, curl_max);
|
||||
let index_curl = self.get_gesture_hover(
|
||||
lerp_pose(index_metacarpal, index_proximal, 0.5),
|
||||
palm_depth,
|
||||
index_tip,
|
||||
index_rad,
|
||||
curl_min,
|
||||
curl_max,
|
||||
);
|
||||
let middle_curl = self.get_gesture_hover(
|
||||
lerp_pose(middle_metacarpal, middle_proximal, 0.5),
|
||||
palm_depth,
|
||||
middle_tip,
|
||||
middle_rad,
|
||||
curl_min,
|
||||
curl_max,
|
||||
);
|
||||
let ring_curl = self.get_gesture_hover(
|
||||
lerp_pose(ring_metacarpal, ring_proximal, 0.5),
|
||||
palm_depth,
|
||||
ring_tip,
|
||||
ring_rad,
|
||||
curl_min,
|
||||
curl_max,
|
||||
);
|
||||
let little_curl = self.get_gesture_hover(
|
||||
lerp_pose(little_metacarpal, little_proximal, 0.5),
|
||||
palm_depth,
|
||||
little_tip,
|
||||
little_rad,
|
||||
curl_min,
|
||||
curl_max,
|
||||
);
|
||||
|
||||
// Grip
|
||||
let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0;
|
||||
let grip_active = grip_curl > 0.0;
|
||||
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::GripCurl,
|
||||
active: grip_active,
|
||||
clicked: grip_curl == 1.0,
|
||||
touching: grip_curl > 0.0,
|
||||
value: grip_curl,
|
||||
});
|
||||
|
||||
// Joystick
|
||||
let joystick_range = config.joystick_range * 0.01;
|
||||
let joystick_center = lerp_pose(index_intermediate, index_distal, 0.5);
|
||||
|
||||
let joystick_up = joystick_center
|
||||
.orientation
|
||||
.mul_vec3(if device_id == *LEFT_HAND_ID {
|
||||
Vec3::X
|
||||
} else {
|
||||
Vec3::NEG_X
|
||||
});
|
||||
let joystick_horizontal_vec =
|
||||
index_intermediate
|
||||
.orientation
|
||||
.mul_vec3(if device_id == *LEFT_HAND_ID {
|
||||
Vec3::Y
|
||||
} else {
|
||||
Vec3::NEG_Y
|
||||
});
|
||||
let joystick_vertical_vec = index_intermediate.orientation.mul_vec3(Vec3::Z);
|
||||
|
||||
let joystick_pos = self.get_joystick_values(
|
||||
joystick_center,
|
||||
thumb_tip,
|
||||
joystick_range,
|
||||
joystick_horizontal_vec,
|
||||
joystick_vertical_vec,
|
||||
config.joystick_offset_horizontal * 0.01,
|
||||
config.joystick_offset_vertical * 0.01,
|
||||
);
|
||||
let joystick_contact = index_curl >= 0.75
|
||||
&& grip_curl > 0.5
|
||||
&& joystick_center.position.distance(thumb_tip.position) <= joystick_range * 3.0
|
||||
&& (thumb_tip.position - joystick_center.position).dot(joystick_up)
|
||||
/ joystick_up.length()
|
||||
<= joystick_range * 2.0;
|
||||
|
||||
let joystick_deadzone: f32 = config.joystick_deadzone * 0.01;
|
||||
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::ThumbCurl,
|
||||
active: thumb_curl >= 0.0,
|
||||
touching: thumb_curl >= 0.0,
|
||||
clicked: thumb_curl >= 0.5,
|
||||
value: thumb_curl,
|
||||
});
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::JoystickX,
|
||||
active: joystick_contact,
|
||||
touching: joystick_contact,
|
||||
clicked: false,
|
||||
value: if joystick_contact && joystick_pos.x.abs() >= joystick_deadzone {
|
||||
joystick_pos.x
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
});
|
||||
gestures.push(HandGesture {
|
||||
id: HandGestureId::JoystickY,
|
||||
active: joystick_contact,
|
||||
touching: joystick_contact,
|
||||
clicked: false,
|
||||
value: if joystick_contact && joystick_pos.y.abs() >= joystick_deadzone {
|
||||
joystick_pos.y
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
});
|
||||
|
||||
gestures
|
||||
}
|
||||
|
||||
fn is_gesture_active(
|
||||
&mut self,
|
||||
gesture_id: HandGestureId,
|
||||
first_anchor: Pose,
|
||||
first_radius: f32,
|
||||
second_anchor: Pose,
|
||||
second_radius: f32,
|
||||
activation_dist: f32,
|
||||
repeat_delay: u32,
|
||||
in_delay: u32,
|
||||
out_delay: u32,
|
||||
device_id: u64,
|
||||
) -> bool {
|
||||
let in_range = first_anchor.position.distance(second_anchor.position)
|
||||
< (activation_dist + first_radius + second_radius);
|
||||
|
||||
let gesture_data = if device_id == *LEFT_HAND_ID {
|
||||
&mut self.gesture_data_left
|
||||
} else {
|
||||
&mut self.gesture_data_right
|
||||
};
|
||||
|
||||
if !gesture_data.contains_key(&gesture_id) {
|
||||
gesture_data.insert(
|
||||
gesture_id,
|
||||
GestureAction {
|
||||
last_activated: 0,
|
||||
last_deactivated: 0,
|
||||
entering: false,
|
||||
entering_since: 0,
|
||||
exiting: false,
|
||||
exiting_since: 0,
|
||||
active: false,
|
||||
},
|
||||
);
|
||||
}
|
||||
let g: &mut GestureAction = gesture_data.get_mut(&gesture_id).unwrap();
|
||||
|
||||
// Disable entering/exiting state if we leave/enter range
|
||||
if in_range {
|
||||
g.exiting = false;
|
||||
} else {
|
||||
g.entering = false;
|
||||
}
|
||||
|
||||
// Get current time, for comparison
|
||||
let time_millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::from_millis(0))
|
||||
.as_millis();
|
||||
|
||||
// Transitioning from inactive to active
|
||||
if in_range && !g.active {
|
||||
// Don't transition state unless the duration of repeat_delay has passed since last deactivation
|
||||
if g.last_deactivated < time_millis - u128::from(repeat_delay) {
|
||||
if g.entering {
|
||||
// Don't transition state unless gesture has been in range for the duration of in_delay
|
||||
if g.entering_since < time_millis - u128::from(in_delay) {
|
||||
g.last_activated = time_millis;
|
||||
g.entering = false;
|
||||
g.active = true;
|
||||
}
|
||||
} else {
|
||||
// Begin tracking entering state
|
||||
g.entering = true;
|
||||
g.entering_since = time_millis;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Transitioning from inactive to active
|
||||
if !in_range && g.active {
|
||||
if g.exiting {
|
||||
// Don't transition state unless gesture has been out of range for the duration of out_delay
|
||||
if g.exiting_since < time_millis - u128::from(out_delay) {
|
||||
g.last_deactivated = time_millis;
|
||||
g.exiting = false;
|
||||
g.active = false;
|
||||
}
|
||||
} else {
|
||||
// Begin tracking exiting state
|
||||
g.exiting = true;
|
||||
g.exiting_since = time_millis;
|
||||
}
|
||||
}
|
||||
|
||||
g.active
|
||||
}
|
||||
|
||||
fn test_gesture_dist(
|
||||
&self,
|
||||
first_anchor: Pose,
|
||||
first_radius: f32,
|
||||
second_anchor: Pose,
|
||||
second_radius: f32,
|
||||
activation_dist: f32,
|
||||
) -> bool {
|
||||
first_anchor.position.distance(second_anchor.position)
|
||||
< (activation_dist + first_radius + second_radius)
|
||||
}
|
||||
|
||||
fn get_gesture_hover(
|
||||
&self,
|
||||
first_anchor: Pose,
|
||||
first_radius: f32,
|
||||
second_anchor: Pose,
|
||||
second_radius: f32,
|
||||
min_dist: f32,
|
||||
max_dist: f32,
|
||||
) -> f32 {
|
||||
(1.0 - (first_anchor.position.distance(second_anchor.position)
|
||||
- min_dist
|
||||
- first_radius
|
||||
- second_radius)
|
||||
/ (max_dist + first_radius + second_radius))
|
||||
.clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
fn get_joystick_values(
|
||||
&self,
|
||||
center: Pose,
|
||||
anchor: Pose,
|
||||
joy_radius: f32,
|
||||
hori_vec: Vec3,
|
||||
vert_vec: Vec3,
|
||||
offset_hori: f32,
|
||||
offset_vert: f32,
|
||||
) -> Vec2 {
|
||||
let x = (anchor.position - center.position).dot(hori_vec) / hori_vec.length() + offset_hori;
|
||||
|
||||
let y = (anchor.position - center.position).dot(vert_vec) / vert_vec.length() + offset_vert;
|
||||
|
||||
Vec2 {
|
||||
x: (x / joy_radius).clamp(-1.0, 1.0),
|
||||
y: (y / joy_radius).clamp(-1.0, 1.0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_click_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option<u64> {
|
||||
if device_id == *LEFT_HAND_ID {
|
||||
match gesture_id {
|
||||
HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_CLICK_ID),
|
||||
HandGestureId::ThumbMiddlePinch => Some(*LEFT_Y_CLICK_ID),
|
||||
HandGestureId::ThumbRingPinch => Some(*LEFT_X_CLICK_ID),
|
||||
HandGestureId::ThumbLittlePinch => Some(*LEFT_MENU_CLICK_ID),
|
||||
HandGestureId::GripCurl => Some(*LEFT_SQUEEZE_CLICK_ID),
|
||||
HandGestureId::ThumbCurl => Some(*LEFT_THUMBSTICK_CLICK_ID),
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
match gesture_id {
|
||||
HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_CLICK_ID),
|
||||
HandGestureId::ThumbMiddlePinch => Some(*RIGHT_B_CLICK_ID),
|
||||
HandGestureId::ThumbRingPinch => Some(*RIGHT_A_CLICK_ID),
|
||||
HandGestureId::GripCurl => Some(*RIGHT_SQUEEZE_CLICK_ID),
|
||||
HandGestureId::ThumbCurl => Some(*RIGHT_THUMBSTICK_CLICK_ID),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_touch_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option<u64> {
|
||||
if device_id == *LEFT_HAND_ID {
|
||||
match gesture_id {
|
||||
HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_TOUCH_ID),
|
||||
HandGestureId::ThumbMiddlePinch => Some(*LEFT_Y_TOUCH_ID),
|
||||
HandGestureId::ThumbRingPinch => Some(*LEFT_X_TOUCH_ID),
|
||||
HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_TOUCH_ID),
|
||||
HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_TOUCH_ID),
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
match gesture_id {
|
||||
HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_TOUCH_ID),
|
||||
HandGestureId::ThumbMiddlePinch => Some(*RIGHT_B_TOUCH_ID),
|
||||
HandGestureId::ThumbRingPinch => Some(*RIGHT_A_TOUCH_ID),
|
||||
HandGestureId::JoystickX => Some(*RIGHT_THUMBSTICK_TOUCH_ID),
|
||||
HandGestureId::JoystickY => Some(*RIGHT_THUMBSTICK_TOUCH_ID),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option<u64> {
|
||||
if device_id == *LEFT_HAND_ID {
|
||||
match gesture_id {
|
||||
HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_VALUE_ID),
|
||||
HandGestureId::GripCurl => Some(*LEFT_SQUEEZE_VALUE_ID),
|
||||
HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_X_ID),
|
||||
HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_Y_ID),
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
match gesture_id {
|
||||
HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_VALUE_ID),
|
||||
HandGestureId::GripCurl => Some(*RIGHT_SQUEEZE_VALUE_ID),
|
||||
HandGestureId::JoystickX => Some(*RIGHT_THUMBSTICK_X_ID),
|
||||
HandGestureId::JoystickY => Some(*RIGHT_THUMBSTICK_Y_ID),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn trigger_hand_gesture_actions(
|
||||
button_mapping_manager: &mut ButtonMappingManager,
|
||||
device_id: u64,
|
||||
gestures: &Vec<HandGesture>,
|
||||
) {
|
||||
for gesture in gestures.iter() {
|
||||
// Click bind
|
||||
let click_bind = get_click_bind_for_gesture(device_id, gesture.id);
|
||||
if click_bind.is_some() {
|
||||
button_mapping_manager.report_button(
|
||||
click_bind.unwrap(),
|
||||
ButtonValue::Binary(gesture.active && gesture.clicked),
|
||||
);
|
||||
}
|
||||
|
||||
// Touch bind
|
||||
let touch_bind = get_touch_bind_for_gesture(device_id, gesture.id);
|
||||
if touch_bind.is_some() {
|
||||
button_mapping_manager.report_button(
|
||||
touch_bind.unwrap(),
|
||||
ButtonValue::Binary(gesture.active && gesture.touching),
|
||||
);
|
||||
}
|
||||
|
||||
// Hover bind
|
||||
let hover_bind = get_hover_bind_for_gesture(device_id, gesture.id);
|
||||
if hover_bind.is_some() {
|
||||
button_mapping_manager.report_button(
|
||||
hover_bind.unwrap(),
|
||||
ButtonValue::Scalar(if gesture.active { gesture.value } else { 0.0 }),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
mod bitrate;
|
||||
mod connection;
|
||||
mod face_tracking;
|
||||
mod hand_gestures;
|
||||
mod haptics;
|
||||
mod input_mapping;
|
||||
mod logging_backend;
|
||||
|
|
|
|||
|
|
@ -659,6 +659,8 @@ pub struct ControllersConfig {
|
|||
|
||||
pub button_mapping_config: AutomaticButtonMappingConfig,
|
||||
|
||||
pub hand_tracking: HandTrackingConfig,
|
||||
|
||||
#[schema(strings(
|
||||
display_name = "Prediction",
|
||||
help = r"Higher values make the controllers track smoother.
|
||||
|
|
@ -700,6 +702,93 @@ Currently this cannot be reliably estimated automatically. The correct value sho
|
|||
pub haptics: Switch<HapticsConfig>,
|
||||
}
|
||||
|
||||
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
|
||||
#[schema(collapsible)]
|
||||
pub struct HandTrackingConfig {
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "Enabling this allows using hand gestures to emulate controller inputs."
|
||||
))]
|
||||
pub use_gestures: Switch<HandGestureConfig>,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "Enabling this passes skeletal hand data (finger tracking) to SteamVR."
|
||||
))]
|
||||
pub enable_skeleton: bool,
|
||||
}
|
||||
|
||||
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
|
||||
pub struct HandGestureConfig {
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How close the tips of your fingers need to be to register a pinch click."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)), suffix = "cm")]
|
||||
pub pinch_touch_distance: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How close together the tips of your fingers need to be to start registering a pinch trigger pull."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0.0, max = 2.5, step = 0.025)), suffix = "cm")]
|
||||
pub pinch_trigger_distance: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How close to your palm the tips of your fingers need to be to register a curl click."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0.0, max = 5.0)), suffix = "cm")]
|
||||
pub curl_touch_distance: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How close to your palm the tips of your fingers need to be to start registering a trigger pull."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0.0, max = 10.0)), suffix = "cm")]
|
||||
pub curl_trigger_distance: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(gui(slider(min = 0.0, max = 100.0)), suffix = "%")]
|
||||
pub joystick_deadzone: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(gui(slider(min = -5.0, max = 5.0)), suffix = "cm")]
|
||||
pub joystick_offset_horizontal: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(gui(slider(min = -5.0, max = 5.0)), suffix = "cm")]
|
||||
pub joystick_offset_vertical: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "The radius of motion of the joystick. The joystick can be controlled if the thumb is within 2x this range."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0.0, max = 5.0)), suffix = "cm")]
|
||||
pub joystick_range: f32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How long the gesture must be continuously held before it is activated."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")]
|
||||
pub activation_delay: u32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How long the gesture must be continuously released before it is deactivated."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")]
|
||||
pub deactivation_delay: u32,
|
||||
|
||||
#[schema(flag = "real-time")]
|
||||
#[schema(strings(
|
||||
help = "How long the after the gesture has been deactivated before it can be activated again."
|
||||
))]
|
||||
#[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")]
|
||||
pub repeat_delay: u32,
|
||||
}
|
||||
|
||||
#[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy)]
|
||||
pub enum PositionRecenteringMode {
|
||||
Disabled,
|
||||
|
|
@ -1232,6 +1321,26 @@ pub fn session_settings_default() -> SettingsDefault {
|
|||
},
|
||||
force_threshold: 0.8,
|
||||
},
|
||||
hand_tracking: HandTrackingConfigDefault {
|
||||
gui_collapsed: true,
|
||||
use_gestures: SwitchDefault {
|
||||
enabled: true,
|
||||
content: HandGestureConfigDefault {
|
||||
pinch_touch_distance: 0.0,
|
||||
pinch_trigger_distance: 0.25,
|
||||
curl_touch_distance: 2.0,
|
||||
curl_trigger_distance: 2.5,
|
||||
joystick_deadzone: 40.0,
|
||||
joystick_offset_horizontal: 0.0,
|
||||
joystick_offset_vertical: 0.0,
|
||||
joystick_range: 1.0,
|
||||
repeat_delay: 100,
|
||||
activation_delay: 50,
|
||||
deactivation_delay: 100,
|
||||
},
|
||||
},
|
||||
enable_skeleton: true,
|
||||
},
|
||||
steamvr_pipeline_frames: 3.0,
|
||||
linear_velocity_cutoff: 0.05,
|
||||
angular_velocity_cutoff: 10.0,
|
||||
|
|
|
|||
Loading…
Reference in New Issue