From cbb11c70da08545a7a4da88fd356bd1f50b7da3f Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 24 Jan 2025 17:10:49 -0700 Subject: [PATCH 1/8] Fix judder regression --- alvr/server_core/src/tracking/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index e60ce0e395..303fc340c5 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -35,9 +35,9 @@ use std::{ }; const DEG_TO_RAD: f32 = PI / 180.0; -// A small history size is used, we just need to cover the time from packet received to data -// processed. -const MAX_HISTORY_SIZE: usize = 16; +// In theory we just need to cover the time from packet received to data +// processed, but for consistency we'll set it to be the same as the client +const MAX_HISTORY_SIZE: usize = 1024; #[derive(Debug)] pub enum HandType { @@ -239,6 +239,13 @@ impl TrackingManager { self.device_motions_history .get(&device_id) .and_then(|motions| { + // TODO(shinyquagsire23): There's a logic bug in here, hackfix for just HMDs for now. + if device_id == *HEAD_ID { + return motions + .iter() + .find(|(timestamp, _)| *timestamp == sample_timestamp) + .map(|(_, motion)| *motion); + } // Get first element to initialize a valid motion reference if let Some((_, motion)) = motions.front() { let mut best_timestamp_diff = Duration::MAX; From 5e2f50edc602abb2df21b92960216d6612bf083e Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 24 Jan 2025 17:21:20 -0700 Subject: [PATCH 2/8] Actual fix lol --- alvr/server_core/src/tracking/mod.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 303fc340c5..a6251d1149 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -239,13 +239,6 @@ impl TrackingManager { self.device_motions_history .get(&device_id) .and_then(|motions| { - // TODO(shinyquagsire23): There's a logic bug in here, hackfix for just HMDs for now. - if device_id == *HEAD_ID { - return motions - .iter() - .find(|(timestamp, _)| *timestamp == sample_timestamp) - .map(|(_, motion)| *motion); - } // Get first element to initialize a valid motion reference if let Some((_, motion)) = motions.front() { let mut best_timestamp_diff = Duration::MAX; @@ -254,7 +247,7 @@ impl TrackingManager { // Note: we are iterating from most recent to oldest for (ts, m) in motions { match ts.cmp(&sample_timestamp) { - Ordering::Equal => return Some(*best_motion_ref), + Ordering::Equal => return Some(*m), Ordering::Greater => { let diff = ts.saturating_sub(sample_timestamp); if diff < best_timestamp_diff { From 05b04ae24af97c1a696f8c2b709b9a1ba68c5c4a Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 24 Jan 2025 17:42:09 -0700 Subject: [PATCH 3/8] Use statistics history size --- alvr/server_core/src/tracking/mod.rs | 33 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index a6251d1149..975a1f3b3c 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -22,8 +22,8 @@ use alvr_common::{ use alvr_events::{EventType, TrackingEvent}; use alvr_packets::{FaceData, Tracking}; use alvr_session::{ - settings_schema::Switch, BodyTrackingConfig, HeadsetConfig, PositionRecenteringMode, - RotationRecenteringMode, Settings, VMCConfig, + settings_schema::Switch, BodyTrackingConfig, ConnectionConfig, HeadsetConfig, + PositionRecenteringMode, RotationRecenteringMode, Settings, VMCConfig, }; use alvr_sockets::StreamReceiver; use std::{ @@ -35,9 +35,6 @@ use std::{ }; const DEG_TO_RAD: f32 = PI / 180.0; -// In theory we just need to cover the time from packet received to data -// processed, but for consistency we'll set it to be the same as the client -const MAX_HISTORY_SIZE: usize = 1024; #[derive(Debug)] pub enum HandType { @@ -124,7 +121,8 @@ impl TrackingManager { // Performs all kinds of tracking transformations, driven by settings. pub fn report_device_motions( &mut self, - config: &HeadsetConfig, + headset_config: &HeadsetConfig, + connection_config: &ConnectionConfig, timestamp: Duration, device_motions: &[(u64, DeviceMotion)], ) { @@ -141,7 +139,7 @@ impl TrackingManager { (*BODY_RIGHT_FOOT_ID, MotionConfig::default()), ]); - if let Switch::Enabled(controllers) = &config.controllers { + if let Switch::Enabled(controllers) = &headset_config.controllers { let t = controllers.left_controller_position_offset; let r = controllers.left_controller_rotation_offset; @@ -219,7 +217,7 @@ impl TrackingManager { if let Some(motions) = self.device_motions_history.get_mut(&device_id) { motions.push_front((timestamp, motion)); - if motions.len() > MAX_HISTORY_SIZE { + if motions.len() > connection_config.statistics_history_size { motions.pop_back(); } } else { @@ -269,6 +267,7 @@ impl TrackingManager { pub fn report_hand_skeleton( &mut self, hand_type: HandType, + connection_config: &ConnectionConfig, timestamp: Duration, mut skeleton: [Pose; 26], ) { @@ -280,7 +279,7 @@ impl TrackingManager { skeleton_history.push_back((timestamp, skeleton)); - if skeleton_history.len() > MAX_HISTORY_SIZE { + if skeleton_history.len() > connection_config.statistics_history_size { skeleton_history.pop_front(); } } @@ -398,6 +397,7 @@ pub fn tracking_loop( let mut tracking_manager_lock = ctx.tracking_manager.write(); let session_manager_lock = SESSION_MANAGER.read(); let headset_config = &session_manager_lock.settings().headset; + let connection_config = &session_manager_lock.settings().connection; let device_motion_keys = tracking .device_motions @@ -407,15 +407,26 @@ pub fn tracking_loop( tracking_manager_lock.report_device_motions( headset_config, + connection_config, timestamp, &tracking.device_motions, ); if let Some(skeleton) = tracking.hand_skeletons[0] { - tracking_manager_lock.report_hand_skeleton(HandType::Left, timestamp, skeleton); + tracking_manager_lock.report_hand_skeleton( + HandType::Left, + connection_config, + timestamp, + skeleton, + ); } if let Some(skeleton) = tracking.hand_skeletons[1] { - tracking_manager_lock.report_hand_skeleton(HandType::Right, timestamp, skeleton); + tracking_manager_lock.report_hand_skeleton( + HandType::Right, + connection_config, + timestamp, + skeleton, + ); } tracking_manager_lock.report_face_data(tracking.face_data); From c56b6b486f88061963daf66b1abffdf3ce20be6c Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 24 Jan 2025 18:14:54 -0700 Subject: [PATCH 4/8] Prevent client timestamps from running backwards --- alvr/client_core/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index d687355b22..c8bb951532 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -242,6 +242,9 @@ impl ClientCoreContext { poll_timestamp }; + // Guarantee that sent timestamps never go backwards by sending the poll time + let reported_timestamp = poll_timestamp; + for (id, motion) in &mut device_motions { let velocity_multiplier = *self.connection_context.velocities_multiplier.read(); motion.linear_velocity *= velocity_multiplier; @@ -252,7 +255,7 @@ impl ClientCoreContext { let mut head_pose_queue = self.connection_context.head_pose_queue.write(); - head_pose_queue.push_back((target_timestamp, motion.pose)); + head_pose_queue.push_back((reported_timestamp, motion.pose)); while head_pose_queue.len() > 1024 { head_pose_queue.pop_front(); @@ -299,7 +302,7 @@ impl ClientCoreContext { if let Some(sender) = &mut *self.connection_context.tracking_sender.lock() { sender .send_header(&Tracking { - target_timestamp, + reported_timestamp, device_motions, hand_skeletons, face_data, @@ -307,7 +310,7 @@ impl ClientCoreContext { .ok(); if let Some(stats) = &mut *self.connection_context.statistics_manager.lock() { - stats.report_input_acquired(target_timestamp); + stats.report_input_acquired(reported_timestamp); } } } From 1c3813b99868fd6408d31c2c75064fc942d3e75a Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 24 Jan 2025 18:21:55 -0700 Subject: [PATCH 5/8] oops --- alvr/client_core/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index c8bb951532..cfb5efd55e 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -302,7 +302,7 @@ impl ClientCoreContext { if let Some(sender) = &mut *self.connection_context.tracking_sender.lock() { sender .send_header(&Tracking { - reported_timestamp, + target_timestamp: reported_timestamp, device_motions, hand_skeletons, face_data, From 578f173fd61cc5bb1c2d9a0a7fcb427f0365ff5e Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Sat, 25 Jan 2025 12:45:01 -0700 Subject: [PATCH 6/8] History maximum once on init --- alvr/server_core/src/tracking/mod.rs | 33 +++++++++++----------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 975a1f3b3c..1f80facdce 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -22,8 +22,8 @@ use alvr_common::{ use alvr_events::{EventType, TrackingEvent}; use alvr_packets::{FaceData, Tracking}; use alvr_session::{ - settings_schema::Switch, BodyTrackingConfig, ConnectionConfig, HeadsetConfig, - PositionRecenteringMode, RotationRecenteringMode, Settings, VMCConfig, + settings_schema::Switch, BodyTrackingConfig, HeadsetConfig, PositionRecenteringMode, + RotationRecenteringMode, Settings, VMCConfig, }; use alvr_sockets::StreamReceiver; use std::{ @@ -57,16 +57,23 @@ pub struct TrackingManager { device_motions_history: HashMap>, hand_skeletons_history: [VecDeque<(Duration, [Pose; 26])>; 2], last_face_data: FaceData, + max_history_size: usize, } impl TrackingManager { pub fn new() -> TrackingManager { + let max_history_size = SESSION_MANAGER + .read() + .settings() + .connection + .statistics_history_size; TrackingManager { last_head_pose: Pose::default(), inverse_recentering_origin: Pose::default(), device_motions_history: HashMap::new(), hand_skeletons_history: [VecDeque::new(), VecDeque::new()], last_face_data: FaceData::default(), + max_history_size: max_history_size, } } @@ -122,7 +129,6 @@ impl TrackingManager { pub fn report_device_motions( &mut self, headset_config: &HeadsetConfig, - connection_config: &ConnectionConfig, timestamp: Duration, device_motions: &[(u64, DeviceMotion)], ) { @@ -217,7 +223,7 @@ impl TrackingManager { if let Some(motions) = self.device_motions_history.get_mut(&device_id) { motions.push_front((timestamp, motion)); - if motions.len() > connection_config.statistics_history_size { + if motions.len() > self.max_history_size { motions.pop_back(); } } else { @@ -267,7 +273,6 @@ impl TrackingManager { pub fn report_hand_skeleton( &mut self, hand_type: HandType, - connection_config: &ConnectionConfig, timestamp: Duration, mut skeleton: [Pose; 26], ) { @@ -279,7 +284,7 @@ impl TrackingManager { skeleton_history.push_back((timestamp, skeleton)); - if skeleton_history.len() > connection_config.statistics_history_size { + if skeleton_history.len() > self.max_history_size { skeleton_history.pop_front(); } } @@ -397,7 +402,6 @@ pub fn tracking_loop( let mut tracking_manager_lock = ctx.tracking_manager.write(); let session_manager_lock = SESSION_MANAGER.read(); let headset_config = &session_manager_lock.settings().headset; - let connection_config = &session_manager_lock.settings().connection; let device_motion_keys = tracking .device_motions @@ -407,26 +411,15 @@ pub fn tracking_loop( tracking_manager_lock.report_device_motions( headset_config, - connection_config, timestamp, &tracking.device_motions, ); if let Some(skeleton) = tracking.hand_skeletons[0] { - tracking_manager_lock.report_hand_skeleton( - HandType::Left, - connection_config, - timestamp, - skeleton, - ); + tracking_manager_lock.report_hand_skeleton(HandType::Left, timestamp, skeleton); } if let Some(skeleton) = tracking.hand_skeletons[1] { - tracking_manager_lock.report_hand_skeleton( - HandType::Right, - connection_config, - timestamp, - skeleton, - ); + tracking_manager_lock.report_hand_skeleton(HandType::Right, timestamp, skeleton); } tracking_manager_lock.report_face_data(tracking.face_data); From 8c8ee3ef46f8f3dd7bb1fd3b2463d1b6ce99f5de Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Sat, 25 Jan 2025 12:57:48 -0700 Subject: [PATCH 7/8] oops, it deadlocked --- alvr/server_core/src/connection.rs | 3 ++- alvr/server_core/src/lib.rs | 4 +++- alvr/server_core/src/tracking/mod.rs | 7 +------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index 9c022629ba..fc3765c044 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -1012,7 +1012,8 @@ fn connection_pipeline( thread::spawn(|| ()) }; - *ctx.tracking_manager.write() = TrackingManager::new(); + *ctx.tracking_manager.write() = + TrackingManager::new(initial_settings.connection.statistics_history_size); let hand_gesture_manager = Arc::new(Mutex::new(HandGestureManager::new())); let tracking_receive_thread = thread::spawn({ diff --git a/alvr/server_core/src/lib.rs b/alvr/server_core/src/lib.rs index d78efe2833..16af3be03c 100644 --- a/alvr/server_core/src/lib.rs +++ b/alvr/server_core/src/lib.rs @@ -215,7 +215,9 @@ impl ServerCoreContext { events_sender, statistics_manager: RwLock::new(Some(stats)), bitrate_manager: Mutex::new(BitrateManager::new(256, 60.0)), - tracking_manager: RwLock::new(TrackingManager::new()), + tracking_manager: RwLock::new(TrackingManager::new( + initial_settings.connection.statistics_history_size, + )), decoder_config: Mutex::new(None), video_mirror_sender: Mutex::new(None), video_recording_file: Mutex::new(None), diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 1f80facdce..1b926d6c5e 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -61,12 +61,7 @@ pub struct TrackingManager { } impl TrackingManager { - pub fn new() -> TrackingManager { - let max_history_size = SESSION_MANAGER - .read() - .settings() - .connection - .statistics_history_size; + pub fn new(max_history_size: usize) -> TrackingManager { TrackingManager { last_head_pose: Pose::default(), inverse_recentering_origin: Pose::default(), From 67ba460caa9c443c2401b33cfaeaeacb3cbaaead Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Mon, 27 Jan 2025 16:52:55 -0700 Subject: [PATCH 8/8] nit --- alvr/server_core/src/tracking/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/server_core/src/tracking/mod.rs b/alvr/server_core/src/tracking/mod.rs index 1b926d6c5e..a70cfc5774 100644 --- a/alvr/server_core/src/tracking/mod.rs +++ b/alvr/server_core/src/tracking/mod.rs @@ -68,7 +68,7 @@ impl TrackingManager { device_motions_history: HashMap::new(), hand_skeletons_history: [VecDeque::new(), VecDeque::new()], last_face_data: FaceData::default(), - max_history_size: max_history_size, + max_history_size, } }