From 707b41a69f521a822468508664db20b27d1656a5 Mon Sep 17 00:00:00 2001 From: zarik5 Date: Sat, 31 Aug 2024 04:01:05 +0200 Subject: [PATCH] refactor(client_openxr): :recycle: Move extra xr ext wrappers in-tree; show body skeleton in lobby (#2366) --- Cargo.lock | 8 +- alvr/client_core/src/c_api.rs | 2 +- alvr/client_core/src/graphics/lobby.rs | 62 +++- alvr/client_openxr/Cargo.toml | 7 +- .../src/extra_extensions/body_tracking_fb.rs | 147 +++++++++ .../extra_extensions/eye_tracking_social.rs | 87 ++++++ .../src/extra_extensions/face_tracking2_fb.rs | 114 +++++++ .../extra_extensions/facial_tracking_htc.rs | 105 +++++++ .../client_openxr/src/extra_extensions/mod.rs | 68 ++++ alvr/client_openxr/src/graphics.rs | 16 +- alvr/client_openxr/src/interaction.rs | 291 ++++++++++-------- alvr/client_openxr/src/lib.rs | 34 +- alvr/client_openxr/src/lobby.rs | 15 + alvr/client_openxr/src/stream.rs | 37 +-- alvr/server_core/src/connection.rs | 17 +- alvr/session/src/settings.rs | 15 +- 16 files changed, 813 insertions(+), 212 deletions(-) create mode 100644 alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs create mode 100644 alvr/client_openxr/src/extra_extensions/eye_tracking_social.rs create mode 100644 alvr/client_openxr/src/extra_extensions/face_tracking2_fb.rs create mode 100644 alvr/client_openxr/src/extra_extensions/facial_tracking_htc.rs create mode 100644 alvr/client_openxr/src/extra_extensions/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 9015121258..1b202b0a32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4089,8 +4089,8 @@ dependencies = [ [[package]] name = "openxr" -version = "0.17.1" -source = "git+https://github.com/alvr-org/openxrs?rev=f1f47ff#f1f47ff8509801bf28d44538cd4a379dc1b686cf" +version = "0.19.0" +source = "git+https://github.com/Ralith/openxrs#d8ea5553d52c4bbaefe2537b8d70b8752f73694d" dependencies = [ "libc", "libloading 0.8.4", @@ -4100,8 +4100,8 @@ dependencies = [ [[package]] name = "openxr-sys" -version = "0.9.3" -source = "git+https://github.com/alvr-org/openxrs?rev=f1f47ff#f1f47ff8509801bf28d44538cd4a379dc1b686cf" +version = "0.11.0" +source = "git+https://github.com/Ralith/openxrs#d8ea5553d52c4bbaefe2537b8d70b8752f73694d" dependencies = [ "libc", ] diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs index 6863ed800b..7488b4bfe9 100644 --- a/alvr/client_core/src/c_api.rs +++ b/alvr/client_core/src/c_api.rs @@ -812,7 +812,7 @@ pub unsafe extern "C" fn alvr_render_lobby_opengl(view_inputs: *const AlvrViewIn LOBBY_RENDERER.with_borrow(|renderer| { if let Some(renderer) = renderer { - renderer.render(view_inputs, [(None, None), (None, None)]); + renderer.render(view_inputs, [(None, None), (None, None)], None); } }); } diff --git a/alvr/client_core/src/graphics/lobby.rs b/alvr/client_core/src/graphics/lobby.rs index afd1a4c036..4b63b3a765 100644 --- a/alvr/client_core/src/graphics/lobby.rs +++ b/alvr/client_core/src/graphics/lobby.rs @@ -1,6 +1,6 @@ use super::{GraphicsContext, SDR_FORMAT}; use alvr_common::{ - glam::{Mat4, UVec2, Vec3, Vec4}, + glam::{Mat4, Quat, UVec2, Vec3, Vec4}, Fov, Pose, }; use glyph_brush_layout::{ @@ -53,6 +53,44 @@ const HAND_SKELETON_BONES: [(usize, usize); 19] = [ (24, 25), ]; +const BODY_SKELETON_BONES_FB: [(usize, usize); 30] = [ + // Spine + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (5, 6), + (6, 7), + // Left arm + (5, 8), + (8, 9), + (9, 10), + (10, 11), + (11, 12), + // Right arm + (5, 13), + (13, 14), + (14, 15), + (15, 16), + (16, 17), + // Left leg + (1, 70), + (70, 71), + (71, 72), + (72, 73), + (73, 74), + (74, 75), + (75, 76), + // Right leg + (1, 77), + (77, 78), + (78, 79), + (79, 80), + (80, 81), + (81, 82), + (82, 83), +]; + fn projection_from_fov(fov: Fov) -> Mat4 { const NEAR: f32 = 0.1; @@ -306,7 +344,8 @@ impl LobbyRenderer { pub fn render( &self, view_inputs: [RenderViewInput; 2], - hand_poses: [(Option, Option<[Pose; 26]>); 2], + hand_data: [(Option, Option<[Pose; 26]>); 2], + body_skeleton_fb: Option>>, ) { let mut encoder = self .context @@ -376,7 +415,7 @@ impl LobbyRenderer { // Bind line pipeline and render hands pass.set_pipeline(&self.line_pipeline); - for (maybe_pose, maybe_skeleton) in &hand_poses { + for (maybe_pose, maybe_skeleton) in &hand_data { if let Some(skeleton) = maybe_skeleton { for (joint1_idx, joint2_idx) in HAND_SKELETON_BONES { let j1_pose = skeleton[joint1_idx]; @@ -410,6 +449,23 @@ impl LobbyRenderer { } } } + if let Some(skeleton) = &body_skeleton_fb { + for (joint1_idx, joint2_idx) in BODY_SKELETON_BONES_FB { + if let (Some(Some(j1_pose)), Some(Some(j2_pose))) = + (skeleton.get(joint1_idx), skeleton.get(joint2_idx)) + { + let transform = Mat4::from_scale_rotation_translation( + Vec3::ONE * Vec3::distance(j1_pose.position, j2_pose.position), + Quat::from_rotation_arc( + -Vec3::Z, + (j2_pose.position - j1_pose.position).normalize(), + ), + j1_pose.position, + ); + transform_draw(&mut pass, view_proj * transform, 2); + } + } + } } self.context.queue.submit(Some(encoder.finish())); diff --git a/alvr/client_openxr/Cargo.toml b/alvr/client_openxr/Cargo.toml index 51d424a981..5c71eb2d86 100644 --- a/alvr/client_openxr/Cargo.toml +++ b/alvr/client_openxr/Cargo.toml @@ -15,8 +15,7 @@ alvr_client_core.workspace = true alvr_packets.workspace = true alvr_session.workspace = true -openxr = { git = "https://github.com/alvr-org/openxrs", rev = "f1f47ff" } -# openxr = { path = "../../../../openxrs/openxr" } +openxr = { git = "https://github.com/Ralith/openxrs" } [target.'cfg(target_os = "android")'.dependencies] android-activity = { version = "0.6", features = ["native-activity"] } @@ -25,7 +24,7 @@ libc = "0.2" ndk-context = "0.1" [package.metadata.android] -package = "alvr.client.dev" # Changed for Meta Store +package = "alvr.client.dev" # Changed for Meta Store install_location = "auto" build_targets = ["aarch64-linux-android"] runtime_libs = "../../deps/android_openxr" @@ -125,7 +124,7 @@ value = "vr_only" name = "com.oculus.supportedDevices" # Note: value is changed for the Meta store, which requires an explicit list of platforms. # "all" is required to support Quest 1 which doesn't have newer platform names registered. -value = "all" +value = "all" [[package.metadata.android.application.meta_data]] name = "com.oculus.vr.focusaware" value = "true" diff --git a/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs b/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs new file mode 100644 index 0000000000..f97c3eab7a --- /dev/null +++ b/alvr/client_openxr/src/extra_extensions/body_tracking_fb.rs @@ -0,0 +1,147 @@ +#![allow(dead_code)] + +use alvr_common::{anyhow::Result, once_cell::sync::Lazy, ToAny}; +use openxr::{self as xr, raw, sys}; +use std::ptr; + +pub const META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME: &str = "XR_META_body_tracking_full_body"; +pub const TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META: Lazy = + Lazy::new(|| xr::StructureType::from_raw(1000274000)); +pub const BODY_JOINT_SET_FULL_BODY_META: Lazy = + Lazy::new(|| xr::BodyJointSetFB::from_raw(1000274000)); + +pub const FULL_BODY_JOINT_LEFT_UPPER_LEG_META: usize = 70; +pub const FULL_BODY_JOINT_LEFT_LOWER_LEG_META: usize = 71; +pub const FULL_BODY_JOINT_LEFT_FOOT_ANKLE_TWIST_META: usize = 72; +pub const FULL_BODY_JOINT_LEFT_FOOT_ANKLE_META: usize = 73; +pub const FULL_BODY_JOINT_LEFT_FOOT_SUBTALAR_META: usize = 74; +pub const FULL_BODY_JOINT_LEFT_FOOT_TRANSVERSE_META: usize = 75; +pub const FULL_BODY_JOINT_LEFT_FOOT_BALL_META: usize = 76; +pub const FULL_BODY_JOINT_RIGHT_UPPER_LEG_META: usize = 77; +pub const FULL_BODY_JOINT_RIGHT_LOWER_LEG_META: usize = 78; +pub const FULL_BODY_JOINT_RIGHT_FOOT_ANKLE_TWIST_META: usize = 79; +pub const FULL_BODY_JOINT_RIGHT_FOOT_ANKLE_META: usize = 80; +pub const FULL_BODY_JOINT_RIGHT_FOOT_SUBTALAR_META: usize = 81; +pub const FULL_BODY_JOINT_RIGHT_FOOT_TRANSVERSE_META: usize = 82; +pub const FULL_BODY_JOINT_RIGHT_FOOT_BALL_META: usize = 83; +pub const FULL_BODY_JOINT_COUNT_META: usize = 84; + +#[repr(C)] +struct SystemPropertiesBodyTrackingFullBodyMETA { + ty: xr::StructureType, + next: *mut std::ffi::c_void, + supports_full_body_tracking: sys::Bool32, +} + +impl super::ExtraExtensions { + pub fn supports_body_tracking_fb(&self, instance: &xr::Instance, system: xr::SystemId) -> bool { + self.get_props( + instance, + system, + sys::SystemBodyTrackingPropertiesFB { + ty: sys::SystemBodyTrackingPropertiesFB::TYPE, + next: ptr::null_mut(), + supports_body_tracking: sys::FALSE, + }, + ) + .map(|props| props.supports_body_tracking.into()) + .unwrap_or(false) + } + + pub fn supports_full_body_tracking_meta( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props( + instance, + system, + SystemPropertiesBodyTrackingFullBodyMETA { + ty: *TYPE_SYSTEM_PROPERTIES_BODY_TRACKING_FULL_BODY_META, + next: ptr::null_mut(), + supports_full_body_tracking: sys::FALSE, + }, + ) + .map(|props| props.supports_full_body_tracking.into()) + .unwrap_or(false) + } + + pub fn create_body_tracker_fb( + &self, + session: &xr::Session, + body_joint_set: xr::BodyJointSetFB, + ) -> Result { + let ext_fns = self.ext_functions_ptrs.fb_body_tracking.to_any()?; + + let mut handle = sys::BodyTrackerFB::NULL; + let info = sys::BodyTrackerCreateInfoFB { + ty: sys::BodyTrackerCreateInfoFB::TYPE, + next: ptr::null(), + body_joint_set, + }; + unsafe { + super::to_any((ext_fns.create_body_tracker)( + session.as_raw(), + &info, + &mut handle, + ))? + }; + + Ok(BodyTrackerFB { handle, ext_fns }) + } +} + +pub struct BodyTrackerFB { + handle: sys::BodyTrackerFB, + ext_fns: raw::BodyTrackingFB, +} + +impl BodyTrackerFB { + pub fn locate_body_joints( + &self, + time: xr::Time, + reference_space: &xr::Space, + joint_count: usize, + ) -> Result>> { + let locate_info = sys::BodyJointsLocateInfoFB { + ty: sys::BodyJointsLocateInfoFB::TYPE, + next: ptr::null(), + base_space: reference_space.as_raw(), + time, + }; + let mut locations = Vec::with_capacity(joint_count); + let mut location_info = sys::BodyJointLocationsFB { + ty: sys::BodyJointLocationsFB::TYPE, + next: ptr::null_mut(), + is_active: sys::FALSE, + confidence: 0.0, + joint_count: joint_count as u32, + joint_locations: locations.as_mut_ptr() as _, + skeleton_changed_count: 0, + time: xr::Time::from_nanos(0), + }; + unsafe { + super::to_any((self.ext_fns.locate_body_joints)( + self.handle, + &locate_info, + &mut location_info, + ))?; + + Ok(if location_info.is_active.into() { + locations.set_len(joint_count); + + Some(locations) + } else { + None + }) + } + } +} + +impl Drop for BodyTrackerFB { + fn drop(&mut self) { + unsafe { + (self.ext_fns.destroy_body_tracker)(self.handle); + } + } +} diff --git a/alvr/client_openxr/src/extra_extensions/eye_tracking_social.rs b/alvr/client_openxr/src/extra_extensions/eye_tracking_social.rs new file mode 100644 index 0000000000..27c6fd2342 --- /dev/null +++ b/alvr/client_openxr/src/extra_extensions/eye_tracking_social.rs @@ -0,0 +1,87 @@ +use alvr_common::{anyhow::Result, ToAny}; +use openxr::{self as xr, raw, sys}; +use std::ptr; + +impl super::ExtraExtensions { + pub fn supports_social_eye_tracking( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props(instance, system, unsafe { + sys::SystemEyeTrackingPropertiesFB::out(ptr::null_mut()).assume_init() + }) + .map(|props| props.supports_eye_tracking.into()) + .unwrap_or(false) + } + + pub fn create_eye_tracker_social( + &self, + session: xr::Session, + ) -> Result { + let ext_fns = self.ext_functions_ptrs.fb_eye_tracking_social.to_any()?; + + let mut handle = sys::EyeTrackerFB::NULL; + let info = sys::EyeTrackerCreateInfoFB { + ty: sys::EyeTrackerCreateInfoFB::TYPE, + next: ptr::null(), + }; + unsafe { + super::to_any((ext_fns.create_eye_tracker)( + session.as_raw(), + &info, + &mut handle, + ))? + }; + + Ok(EyeTrackerSocial { handle, ext_fns }) + } +} + +pub struct EyeTrackerSocial { + handle: sys::EyeTrackerFB, + ext_fns: raw::EyeTrackingSocialFB, +} + +impl EyeTrackerSocial { + pub fn get_eye_gazes( + &self, + base: &xr::Space, + time: xr::Time, + ) -> Result<[Option; 2]> { + let gaze_info = sys::EyeGazesInfoFB { + ty: sys::EyeGazesInfoFB::TYPE, + next: ptr::null(), + base_space: base.as_raw(), + time, + }; + + let mut eye_gazes = sys::EyeGazesFB::out(ptr::null_mut()); + + let eye_gazes = unsafe { + super::to_any((self.ext_fns.get_eye_gazes)( + self.handle, + &gaze_info, + eye_gazes.as_mut_ptr(), + ))?; + + eye_gazes.assume_init() + }; + + let left_valid: bool = eye_gazes.gaze[0].is_valid.into(); + let right_valid: bool = eye_gazes.gaze[1].is_valid.into(); + + Ok([ + left_valid.then(|| eye_gazes.gaze[0].gaze_pose), + right_valid.then(|| eye_gazes.gaze[1].gaze_pose), + ]) + } +} + +impl Drop for EyeTrackerSocial { + fn drop(&mut self) { + unsafe { + (self.ext_fns.destroy_eye_tracker)(self.handle); + } + } +} diff --git a/alvr/client_openxr/src/extra_extensions/face_tracking2_fb.rs b/alvr/client_openxr/src/extra_extensions/face_tracking2_fb.rs new file mode 100644 index 0000000000..c3602db649 --- /dev/null +++ b/alvr/client_openxr/src/extra_extensions/face_tracking2_fb.rs @@ -0,0 +1,114 @@ +use alvr_common::{anyhow::Result, ToAny}; +use openxr::{self as xr, raw, sys}; +use std::ptr; + +impl super::ExtraExtensions { + pub fn supports_fb_visual_face_tracking( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props(instance, system, unsafe { + sys::SystemFaceTrackingProperties2FB::out(ptr::null_mut()).assume_init() + }) + .map(|props| props.supports_visual_face_tracking.into()) + .unwrap_or(false) + } + + pub fn supports_fb_audio_face_tracking( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props(instance, system, unsafe { + sys::SystemFaceTrackingProperties2FB::out(ptr::null_mut()).assume_init() + }) + .map(|props| props.supports_audio_face_tracking.into()) + .unwrap_or(false) + } + + pub fn create_face_tracker2_fb( + &self, + session: &xr::Session, + visual: bool, + audio: bool, + ) -> Result { + let ext_fns = self.ext_functions_ptrs.fb_face_tracking2.to_any()?; + + let mut requested_data_sources = vec![]; + if visual { + requested_data_sources.push(sys::FaceTrackingDataSource2FB::VISUAL); + } + if audio { + requested_data_sources.push(sys::FaceTrackingDataSource2FB::AUDIO); + } + + let mut handle = sys::FaceTracker2FB::NULL; + let info = sys::FaceTrackerCreateInfo2FB { + ty: sys::FaceTrackerCreateInfo2FB::TYPE, + next: ptr::null(), + face_expression_set: xr::FaceExpressionSet2FB::DEFAULT, + requested_data_source_count: requested_data_sources.len() as u32, + requested_data_sources: requested_data_sources.as_mut_ptr(), + }; + unsafe { + super::to_any((ext_fns.create_face_tracker2)( + session.as_raw(), + &info, + &mut handle, + ))? + }; + + Ok(FaceTracker2FB { handle, ext_fns }) + } +} + +pub struct FaceTracker2FB { + handle: sys::FaceTracker2FB, + ext_fns: raw::FaceTracking2FB, +} + +impl FaceTracker2FB { + pub fn get_face_expression_weights(&self, time: xr::Time) -> Result>> { + let expression_info = sys::FaceExpressionInfo2FB { + ty: sys::FaceExpressionInfo2FB::TYPE, + next: ptr::null(), + time, + }; + + let weights_count = xr::FaceExpression2FB::COUNT.into_raw() as usize; + let confidence_count = xr::FaceConfidence2FB::COUNT.into_raw() as usize; + + let mut weights = Vec::with_capacity(weights_count); + let mut confidences = vec![0.0; confidence_count]; + + let mut expression_weights = sys::FaceExpressionWeights2FB { + ty: sys::FaceExpressionWeights2FB::TYPE, + next: ptr::null_mut(), + weight_count: weights_count as u32, + weights: weights.as_mut_ptr() as _, + confidence_count: confidence_count as u32, + confidences: confidences.as_mut_ptr() as _, + is_valid: sys::FALSE, + is_eye_following_blendshapes_valid: sys::FALSE, + data_source: sys::FaceTrackingDataSource2FB::from_raw(0), + time: xr::Time::from_nanos(0), + }; + + unsafe { + super::to_any((self.ext_fns.get_face_expression_weights2)( + self.handle, + &expression_info, + &mut expression_weights, + ))?; + + if expression_weights.is_valid.into() { + weights.set_len(weights_count); + + Ok(Some(weights)) + } else { + Ok(None) + } + } + } +} diff --git a/alvr/client_openxr/src/extra_extensions/facial_tracking_htc.rs b/alvr/client_openxr/src/extra_extensions/facial_tracking_htc.rs new file mode 100644 index 0000000000..a71d0f8f1c --- /dev/null +++ b/alvr/client_openxr/src/extra_extensions/facial_tracking_htc.rs @@ -0,0 +1,105 @@ +use alvr_common::{anyhow::Result, ToAny}; +use openxr::{self as xr, raw, sys}; +use std::ptr; + +impl super::ExtraExtensions { + pub fn supports_htc_eye_facial_tracking( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props(instance, system, unsafe { + sys::SystemFacialTrackingPropertiesHTC::out(ptr::null_mut()).assume_init() + }) + .map(|props| props.support_eye_facial_tracking.into()) + .unwrap_or(false) + } + + pub fn supports_htc_lip_facial_tracking( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props(instance, system, unsafe { + sys::SystemFacialTrackingPropertiesHTC::out(ptr::null_mut()).assume_init() + }) + .map(|props| props.support_lip_facial_tracking.into()) + .unwrap_or(false) + } + + pub fn create_facial_tracker_htc( + &self, + session: &xr::Session, + facial_tracking_type: xr::FacialTrackingTypeHTC, + ) -> Result { + let ext_fns = self.ext_functions_ptrs.htc_facial_tracking.to_any()?; + + let mut handle = sys::FacialTrackerHTC::NULL; + let info = sys::FacialTrackerCreateInfoHTC { + ty: sys::FacialTrackerCreateInfoHTC::TYPE, + next: ptr::null(), + facial_tracking_type, + }; + unsafe { + super::to_any((ext_fns.create_facial_tracker)( + session.as_raw(), + &info, + &mut handle, + ))? + }; + + let expression_count = if facial_tracking_type == sys::FacialTrackingTypeHTC::EYE_DEFAULT { + sys::FACIAL_EXPRESSION_EYE_COUNT_HTC + } else { + sys::FACIAL_EXPRESSION_LIP_COUNT_HTC + }; + + Ok(FacialTrackerHTC { + handle, + ext_fns, + expression_count, + }) + } +} + +pub struct FacialTrackerHTC { + handle: sys::FacialTrackerHTC, + ext_fns: raw::FacialTrackingHTC, + expression_count: usize, +} + +impl FacialTrackerHTC { + pub fn get_facial_expressions(&self) -> Result>> { + let mut weights = Vec::with_capacity(self.expression_count); + + let mut facial_expressions = sys::FacialExpressionsHTC { + ty: sys::FacialExpressionsHTC::TYPE, + next: ptr::null_mut(), + is_active: sys::FALSE, + sample_time: xr::Time::from_nanos(0), + expression_count: self.expression_count as u32, + expression_weightings: weights.as_mut_ptr(), + }; + + unsafe { + super::to_any((self.ext_fns.get_facial_expressions)( + self.handle, + &mut facial_expressions, + ))?; + + if facial_expressions.is_active.into() { + Ok(Some(weights)) + } else { + Ok(None) + } + } + } +} + +impl Drop for FacialTrackerHTC { + fn drop(&mut self) { + unsafe { + (self.ext_fns.destroy_facial_tracker)(self.handle); + } + } +} diff --git a/alvr/client_openxr/src/extra_extensions/mod.rs b/alvr/client_openxr/src/extra_extensions/mod.rs new file mode 100644 index 0000000000..21feefd42c --- /dev/null +++ b/alvr/client_openxr/src/extra_extensions/mod.rs @@ -0,0 +1,68 @@ +mod body_tracking_fb; +mod eye_tracking_social; +mod face_tracking2_fb; +mod facial_tracking_htc; + +pub use body_tracking_fb::*; +pub use eye_tracking_social::*; +pub use face_tracking2_fb::*; +pub use facial_tracking_htc::*; + +use alvr_common::anyhow::{anyhow, Result}; +use openxr::{self as xr, sys}; +use std::ptr; + +fn to_any(result: sys::Result) -> Result<()> { + if result.into_raw() >= 0 { + Ok(()) + } else { + Err(anyhow!("OpenXR error: {:?}", result)) + } +} + +#[derive(Clone)] +pub struct ExtraExtensions { + base_function_ptrs: xr::raw::Instance, + ext_functions_ptrs: xr::InstanceExtensions, +} + +impl ExtraExtensions { + pub fn new(instance: &xr::Instance) -> Self { + Self { + base_function_ptrs: instance.fp().clone(), + ext_functions_ptrs: *instance.exts(), + } + } + + fn get_props( + &self, + instance: &xr::Instance, + system: xr::SystemId, + default_struct: T, + ) -> Option { + let mut props = default_struct; + + let mut system_properties = sys::SystemProperties::out((&mut props as *mut T).cast()); + let result = unsafe { + (self.base_function_ptrs.get_system_properties)( + instance.as_raw(), + system, + system_properties.as_mut_ptr(), + ) + }; + + (result.into_raw() >= 0).then_some(props) + } + + pub fn supports_eye_gaze_interaction( + &self, + instance: &xr::Instance, + system: xr::SystemId, + ) -> bool { + self.get_props(instance, system, unsafe { + sys::SystemEyeGazeInteractionPropertiesEXT::out(ptr::null_mut()).assume_init() + }) + .map(|props| props.supports_eye_gaze_interaction.into()) + .unwrap_or(false) + } +} diff --git a/alvr/client_openxr/src/graphics.rs b/alvr/client_openxr/src/graphics.rs index a5a9e9cc0d..10cc5fe068 100644 --- a/alvr/client_openxr/src/graphics.rs +++ b/alvr/client_openxr/src/graphics.rs @@ -27,6 +27,7 @@ pub fn swapchain_format( graphics::choose_swapchain_format(&formats, enable_hdr) } +#[allow(unused_variables)] pub fn create_swapchain( session: &xr::Session, gfx_ctx: &GraphicsContext, @@ -48,20 +49,7 @@ pub fn create_swapchain( mip_count: 1, }; - if let Some(foveation) = foveation { - let swapchain = session - .create_swapchain_with_foveation( - &swapchain_info, - xr::SwapchainCreateFoveationFlagsFB::SCALED_BIN, - ) - .unwrap(); - - swapchain.update_foveation(foveation).unwrap(); - - swapchain - } else { - session.create_swapchain(&swapchain_info).unwrap() - } + session.create_swapchain(&swapchain_info).unwrap() } // This is needed to work around lifetime limitations diff --git a/alvr/client_openxr/src/interaction.rs b/alvr/client_openxr/src/interaction.rs index a8969dbee6..fbfc425bfa 100644 --- a/alvr/client_openxr/src/interaction.rs +++ b/alvr/client_openxr/src/interaction.rs @@ -1,10 +1,17 @@ -use crate::{from_xr_pose, from_xr_quat, from_xr_vec3, Platform, XrContext}; +use crate::{ + extra_extensions::{ + BodyTrackerFB, EyeTrackerSocial, FaceTracker2FB, FacialTrackerHTC, + BODY_JOINT_SET_FULL_BODY_META, FULL_BODY_JOINT_COUNT_META, + FULL_BODY_JOINT_LEFT_FOOT_BALL_META, FULL_BODY_JOINT_LEFT_LOWER_LEG_META, + FULL_BODY_JOINT_RIGHT_FOOT_BALL_META, FULL_BODY_JOINT_RIGHT_LOWER_LEG_META, + }, + Platform, XrContext, +}; use alvr_common::{glam::Vec3, *}; use alvr_packets::{ButtonEntry, ButtonValue}; use alvr_session::{BodyTrackingSourcesConfig, FaceTrackingSourcesConfig}; use openxr as xr; use std::collections::HashMap; -use xr::sys::FullBodyJointMETA; use xr::SpaceLocationFlags; pub enum ButtonAction { @@ -24,15 +31,14 @@ pub struct HandInteraction { pub struct FaceSources { pub combined_eyes_source: Option<(xr::Action, xr::Space)>, - pub eye_tracker_fb: Option, - pub face_tracker_fb: Option, - pub eye_tracker_htc: Option, - pub lip_tracker_htc: Option, + pub eye_tracker_fb: Option, + pub face_tracker_fb: Option, + pub eye_tracker_htc: Option, + pub lip_tracker_htc: Option, } pub struct BodySources { - pub body_tracker_full_body_meta: Option, - pub enable_full_body: bool, + pub body_tracker_fb: Option<(BodyTrackerFB, usize)>, } pub struct InteractionContext { @@ -49,6 +55,32 @@ pub fn initialize_interaction( face_tracking_sources: Option, body_tracking_sources: Option, ) -> InteractionContext { + // todo: check which permissions are needed for htc + #[cfg(target_os = "android")] + if let Some(config) = &face_tracking_sources { + if (config.combined_eye_gaze || config.eye_tracking_fb) + && matches!(platform, Platform::Quest3 | Platform::QuestPro) + { + alvr_client_core::try_get_permission("com.oculus.permission.EYE_TRACKING") + } + if config.combined_eye_gaze && matches!(platform, Platform::Pico4 | Platform::PicoNeo3) { + alvr_client_core::try_get_permission("com.picovr.permission.EYE_TRACKING") + } + if config.face_tracking_fb && matches!(platform, Platform::Quest3 | Platform::QuestPro) { + alvr_client_core::try_get_permission("android.permission.RECORD_AUDIO"); + alvr_client_core::try_get_permission("com.oculus.permission.FACE_TRACKING") + } + } + + #[cfg(target_os = "android")] + if let Some(config) = &body_tracking_sources { + if (config.body_tracking_fb.enabled()) + && matches!(platform, Platform::Quest3 | Platform::QuestPro) + { + alvr_client_core::try_get_permission("com.oculus.permission.BODY_TRACKING") + } + } + let action_set = xr_ctx .instance .create_action_set("alvr_interaction", "ALVR interaction", 0) @@ -179,11 +211,9 @@ pub fn initialize_interaction( .as_ref() .map(|s| s.combined_eye_gaze) .unwrap_or(false) - && xr_ctx.instance.exts().ext_eye_gaze_interaction.is_some() && xr_ctx - .instance - .supports_eye_gaze_interaction(xr_ctx.system) - .unwrap()) + .extra_extensions + .supports_eye_gaze_interaction(&xr_ctx.instance, xr_ctx.system)) .then(|| { let action = action_set .create_action("combined_eye_gaze", "Combined eye gaze", &[]) @@ -201,7 +231,7 @@ pub fn initialize_interaction( .unwrap(); let space = action - .create_space(xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) + .create_space(&xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) .unwrap(); (action, space) @@ -210,17 +240,17 @@ pub fn initialize_interaction( xr_ctx.session.attach_action_sets(&[&action_set]).unwrap(); let left_grip_space = left_grip_action - .create_space(xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) + .create_space(&xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) .unwrap(); let right_grip_space = right_grip_action - .create_space(xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) + .create_space(&xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) .unwrap(); let left_aim_space = left_aim_action - .create_space(xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) + .create_space(&xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) .unwrap(); let right_aim_space = right_aim_action - .create_space(xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) + .create_space(&xr_ctx.session.clone(), xr::Path::NULL, xr::Posef::IDENTITY) .unwrap(); let (left_hand_tracker, right_hand_tracker) = @@ -242,84 +272,90 @@ pub fn initialize_interaction( .as_ref() .map(|s| s.eye_tracking_fb) .unwrap_or(false) - && xr_ctx.instance.exts().fb_eye_tracking_social.is_some() && xr_ctx - .instance - .supports_social_eye_tracking(xr_ctx.system) - .unwrap()) - .then(|| xr_ctx.session.create_eye_tracker_social().unwrap()); + .extra_extensions + .supports_social_eye_tracking(&xr_ctx.instance, xr_ctx.system)) + .then(|| { + xr_ctx + .extra_extensions + .create_eye_tracker_social(xr_ctx.session.clone()) + .unwrap() + }); let face_tracker_fb = (face_tracking_sources .as_ref() .map(|s| s.face_tracking_fb) .unwrap_or(false) - && xr_ctx.instance.exts().fb_face_tracking2.is_some() && xr_ctx - .instance - .supports_fb_visual_face_tracking(xr_ctx.system) - .unwrap() + .extra_extensions + .supports_fb_visual_face_tracking(&xr_ctx.instance, xr_ctx.system) && xr_ctx - .instance - .supports_fb_audio_face_tracking(xr_ctx.system) - .unwrap()) - .then(|| xr_ctx.session.create_face_tracker2_fb(true, true).unwrap()); + .extra_extensions + .supports_fb_audio_face_tracking(&xr_ctx.instance, xr_ctx.system)) + .then(|| { + xr_ctx + .extra_extensions + .create_face_tracker2_fb(&xr_ctx.session, true, true) + .unwrap() + }); let eye_tracker_htc = (face_tracking_sources .as_ref() .map(|s| s.eye_expressions_htc) .unwrap_or(false) - && xr_ctx.instance.exts().htc_facial_tracking.is_some() && xr_ctx - .instance - .supports_htc_eye_facial_tracking(xr_ctx.system) - .unwrap()) + .extra_extensions + .supports_htc_eye_facial_tracking(&xr_ctx.instance, xr_ctx.system)) .then(|| { xr_ctx - .session - .create_facial_tracker_htc(xr::FacialTrackingTypeHTC::EYE_DEFAULT) + .extra_extensions + .create_facial_tracker_htc(&xr_ctx.session, xr::FacialTrackingTypeHTC::EYE_DEFAULT) .unwrap() }); let lip_tracker_htc = (face_tracking_sources .map(|s| s.lip_expressions_htc) .unwrap_or(false) - && xr_ctx.instance.exts().htc_facial_tracking.is_some() && xr_ctx - .instance - .supports_htc_lip_facial_tracking(xr_ctx.system) - .unwrap()) + .extra_extensions + .supports_htc_lip_facial_tracking(&xr_ctx.instance, xr_ctx.system)) .then(|| { xr_ctx - .session - .create_facial_tracker_htc(xr::FacialTrackingTypeHTC::LIP_DEFAULT) + .extra_extensions + .create_facial_tracker_htc(&xr_ctx.session, xr::FacialTrackingTypeHTC::LIP_DEFAULT) .unwrap() }); - let enable_full_body = body_tracking_sources.clone().is_some_and(|s| { - s.body_tracking_full_body_meta - .into_option() - .is_some_and(|t| t.enable_full_body) - }); + let body_tracker_fb = if let Some(body_tracking_fb) = + body_tracking_sources.and_then(|s| s.body_tracking_fb.into_option()) + { + if body_tracking_fb.full_body + && xr_ctx + .extra_extensions + .supports_full_body_tracking_meta(&xr_ctx.instance, xr_ctx.system) + { + let tracker = xr_ctx + .extra_extensions + .create_body_tracker_fb(&xr_ctx.session, *BODY_JOINT_SET_FULL_BODY_META) + .unwrap(); - let body_tracker_full_body_meta = (body_tracking_sources - .as_ref() - .map(|s| s.body_tracking_full_body_meta.enabled()) - .unwrap_or(false) - && xr_ctx - .instance - .exts() - .meta_body_tracking_full_body - .is_some() - && xr_ctx - .instance - .supports_meta_body_tracking_full_body(xr_ctx.system) - .unwrap()) - .then(|| { - xr_ctx - .session - .create_body_tracker_full_body_meta(enable_full_body) - .unwrap() - }); + Some((tracker, FULL_BODY_JOINT_COUNT_META)) + } else if xr_ctx + .extra_extensions + .supports_body_tracking_fb(&xr_ctx.instance, xr_ctx.system) + { + let tracker = xr_ctx + .extra_extensions + .create_body_tracker_fb(&xr_ctx.session, xr::BodyJointSetFB::DEFAULT) + .unwrap(); + + Some((tracker, xr::BodyJointFB::COUNT.into_raw() as usize)) + } else { + None + } + } else { + None + }; InteractionContext { action_set, @@ -351,10 +387,7 @@ pub fn initialize_interaction( eye_tracker_htc, lip_tracker_htc, }, - body_sources: BodySources { - body_tracker_full_body_meta, - enable_full_body, - }, + body_sources: BodySources { body_tracker_fb }, } } @@ -384,12 +417,12 @@ pub fn get_hand_data( .location_flags .contains(xr::SpaceLocationFlags::POSITION_VALID) { - *last_position = from_xr_vec3(joint_locations[0].pose.position); + *last_position = crate::from_xr_vec3(joint_locations[0].pose.position); } let root_motion = DeviceMotion { pose: Pose { - orientation: from_xr_quat(joint_locations[0].pose.orientation), + orientation: crate::from_xr_quat(joint_locations[0].pose.orientation), position: *last_position, }, linear_velocity: Vec3::ZERO, @@ -398,7 +431,7 @@ pub fn get_hand_data( let joints = joint_locations .iter() - .map(|j| from_xr_pose(j.pose)) + .map(|j| crate::from_xr_pose(j.pose)) .collect::>() .try_into() .unwrap(); @@ -430,16 +463,16 @@ pub fn get_hand_data( .location_flags .contains(xr::SpaceLocationFlags::POSITION_VALID) { - *last_position = from_xr_vec3(location.pose.position); + *last_position = crate::from_xr_vec3(location.pose.position); } let hand_motion = DeviceMotion { pose: Pose { - orientation: from_xr_quat(location.pose.orientation), + orientation: crate::from_xr_quat(location.pose.orientation), position: *last_position, }, - linear_velocity: from_xr_vec3(velocity.linear_velocity), - angular_velocity: from_xr_vec3(velocity.angular_velocity), + linear_velocity: crate::from_xr_vec3(velocity.linear_velocity), + angular_velocity: crate::from_xr_vec3(velocity.angular_velocity), }; (Some(hand_motion), None) @@ -495,8 +528,8 @@ pub fn get_eye_gazes( if let Ok(gazes) = tracker.get_eye_gazes(reference_space, time) { return [ - gazes.gaze[0].as_ref().map(|g| from_xr_pose(g.pose)), - gazes.gaze[1].as_ref().map(|g| from_xr_pose(g.pose)), + gazes[0].map(crate::from_xr_pose), + gazes[1].map(crate::from_xr_pose), ]; } }; @@ -516,7 +549,7 @@ pub fn get_eye_gazes( location .location_flags .contains(xr::SpaceLocationFlags::ORIENTATION_TRACKED) - .then(|| from_xr_pose(location.pose)), + .then(|| crate::from_xr_pose(location.pose)), None, ] } else { @@ -529,7 +562,7 @@ pub fn get_fb_face_expression(context: &FaceSources, time: xr::Time) -> Option Option> { @@ -537,7 +570,7 @@ pub fn get_htc_eye_expression(context: &FaceSources) -> Option> { .eye_tracker_htc .as_ref() .and_then(|t| t.get_facial_expressions().ok().flatten()) - .map(|w| w.weights.into_iter().collect()) + .map(|weights| weights.into_iter().collect()) } pub fn get_htc_lip_expression(context: &FaceSources) -> Option> { @@ -545,17 +578,43 @@ pub fn get_htc_lip_expression(context: &FaceSources) -> Option> { .lip_tracker_htc .as_ref() .and_then(|t| t.get_facial_expressions().ok().flatten()) - .map(|w| w.weights.into_iter().collect()) + .map(|weights| weights.into_iter().collect()) } -pub fn get_meta_body_tracking_full_body_points( +pub fn get_fb_body_skeleton( reference_space: &xr::Space, time: xr::Time, - body_tracker_full_body_meta: &xr::BodyTrackerFullBodyMETA, - full_body: bool, + body_tracker: &BodyTrackerFB, + joint_count: usize, +) -> Option>> { + body_tracker + .locate_body_joints(time, reference_space, joint_count) + .ok() + .flatten() + .map(|joints| { + let valid_flags: SpaceLocationFlags = + SpaceLocationFlags::ORIENTATION_VALID | SpaceLocationFlags::POSITION_VALID; + + joints + .iter() + .map(|joint| { + joint + .location_flags + .contains(valid_flags) + .then(|| crate::from_xr_pose(joint.pose)) + }) + .collect() + }) +} + +pub fn get_fb_body_tracking_points( + reference_space: &xr::Space, + time: xr::Time, + body_tracker: &BodyTrackerFB, + joint_count: usize, ) -> Vec<(u64, DeviceMotion)> { - if let Some(joint_locations) = reference_space - .locate_body_joints_full_body_meta(body_tracker_full_body_meta, time, full_body) + if let Some(joint_locations) = body_tracker + .locate_body_joints(time, reference_space, joint_count) .ok() .flatten() { @@ -564,12 +623,12 @@ pub fn get_meta_body_tracking_full_body_points( let mut joints = Vec::<(u64, DeviceMotion)>::with_capacity(8); - if let Some(joint) = joint_locations.get(FullBodyJointMETA::CHEST.into_raw() as usize) { - if joint.location_flags & valid_flags == valid_flags { + if let Some(joint) = joint_locations.get(xr::BodyJointFB::CHEST.into_raw() as usize) { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_CHEST_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -577,12 +636,12 @@ pub fn get_meta_body_tracking_full_body_points( } } - if let Some(joint) = joint_locations.get(FullBodyJointMETA::HIPS.into_raw() as usize) { - if joint.location_flags & valid_flags == valid_flags { + if let Some(joint) = joint_locations.get(xr::BodyJointFB::HIPS.into_raw() as usize) { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_HIPS_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -591,13 +650,13 @@ pub fn get_meta_body_tracking_full_body_points( } if let Some(joint) = - joint_locations.get(FullBodyJointMETA::LEFT_ARM_LOWER.into_raw() as usize) + joint_locations.get(xr::BodyJointFB::LEFT_ARM_LOWER.into_raw() as usize) { - if joint.location_flags & valid_flags == valid_flags { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_LEFT_ELBOW_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -606,13 +665,13 @@ pub fn get_meta_body_tracking_full_body_points( } if let Some(joint) = - joint_locations.get(FullBodyJointMETA::RIGHT_ARM_LOWER.into_raw() as usize) + joint_locations.get(xr::BodyJointFB::RIGHT_ARM_LOWER.into_raw() as usize) { - if joint.location_flags & valid_flags == valid_flags { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_RIGHT_ELBOW_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -620,14 +679,12 @@ pub fn get_meta_body_tracking_full_body_points( } } - if let Some(joint) = - joint_locations.get(FullBodyJointMETA::LEFT_LOWER_LEG.into_raw() as usize) - { - if joint.location_flags & valid_flags == valid_flags { + if let Some(joint) = joint_locations.get(FULL_BODY_JOINT_LEFT_LOWER_LEG_META) { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_LEFT_KNEE_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -635,14 +692,12 @@ pub fn get_meta_body_tracking_full_body_points( } } - if let Some(joint) = - joint_locations.get(FullBodyJointMETA::LEFT_FOOT_BALL.into_raw() as usize) - { - if joint.location_flags & valid_flags == valid_flags { + if let Some(joint) = joint_locations.get(FULL_BODY_JOINT_LEFT_FOOT_BALL_META) { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_LEFT_FOOT_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -650,14 +705,12 @@ pub fn get_meta_body_tracking_full_body_points( } } - if let Some(joint) = - joint_locations.get(FullBodyJointMETA::RIGHT_LOWER_LEG.into_raw() as usize) - { - if joint.location_flags & valid_flags == valid_flags { + if let Some(joint) = joint_locations.get(FULL_BODY_JOINT_RIGHT_LOWER_LEG_META) { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_RIGHT_KNEE_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, @@ -665,14 +718,12 @@ pub fn get_meta_body_tracking_full_body_points( } } - if let Some(joint) = - joint_locations.get(FullBodyJointMETA::RIGHT_FOOT_BALL.into_raw() as usize) - { - if joint.location_flags & valid_flags == valid_flags { + if let Some(joint) = joint_locations.get(FULL_BODY_JOINT_RIGHT_FOOT_BALL_META) { + if joint.location_flags.contains(valid_flags) { joints.push(( *BODY_RIGHT_FOOT_ID, DeviceMotion { - pose: from_xr_pose(joint.pose), + pose: crate::from_xr_pose(joint.pose), linear_velocity: Vec3::ZERO, angular_velocity: Vec3::ZERO, }, diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs index 46e77ac26f..a41b019961 100644 --- a/alvr/client_openxr/src/lib.rs +++ b/alvr/client_openxr/src/lib.rs @@ -1,4 +1,5 @@ mod c_api; +mod extra_extensions; mod graphics; mod interaction; mod lobby; @@ -13,6 +14,7 @@ use alvr_common::{ glam::{Quat, UVec2, Vec3}, info, Fov, Pose, HAND_LEFT_ID, }; +use extra_extensions::{ExtraExtensions, META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME}; use lobby::Lobby; use openxr as xr; use std::{ @@ -93,6 +95,7 @@ pub struct XrContext { instance: xr::Instance, system: xr::SystemId, session: xr::Session, + extra_extensions: ExtraExtensions, } fn default_view() -> xr::View { @@ -150,7 +153,6 @@ pub fn entry_point() { exts.fb_eye_tracking_social = available_extensions.fb_eye_tracking_social; exts.fb_face_tracking2 = available_extensions.fb_face_tracking2; exts.fb_body_tracking = available_extensions.fb_body_tracking; - exts.meta_body_tracking_full_body = available_extensions.meta_body_tracking_full_body; exts.fb_foveation = available_extensions.fb_foveation; exts.fb_foveation_configuration = available_extensions.fb_foveation_configuration; exts.fb_swapchain_update_state = available_extensions.fb_swapchain_update_state; @@ -163,6 +165,11 @@ pub fn entry_point() { } exts.khr_convert_timespec_time = true; exts.khr_opengl_es_enable = true; + exts.other = available_extensions + .other + .into_iter() + .filter(|ext| [META_BODY_TRACKING_FULL_BODY_EXTENSION_NAME].contains(&ext.as_str())) + .collect(); let available_layers = xr_entry.enumerate_layers().unwrap(); alvr_common::info!("OpenXR available layers: {available_layers:#?}"); @@ -174,6 +181,7 @@ pub fn entry_point() { application_version: 0, engine_name: "ALVR", engine_version: 0, + api_version: xr::Version::new(1, 0, 0), }, &exts, &[], @@ -205,6 +213,7 @@ pub fn entry_point() { instance: xr_instance.clone(), system: xr_system, session: xr_session.clone(), + extra_extensions: ExtraExtensions::new(&xr_instance), }; let views_config = xr_instance @@ -351,19 +360,18 @@ pub fn entry_point() { stream_config = Some(new_config); xr_session.request_exit().ok(); - continue; - } - - stream_context = Some(StreamContext::new( - Arc::clone(&core_context), - xr_context.clone(), - Rc::clone(&graphics_context), - Arc::clone(&interaction_context), - platform, - &new_config, - )); + } else { + stream_context = Some(StreamContext::new( + Arc::clone(&core_context), + xr_context.clone(), + Rc::clone(&graphics_context), + Arc::clone(&interaction_context), + platform, + &new_config, + )); - stream_config = Some(new_config); + stream_config = Some(new_config); + } } ClientCoreEvent::StreamingStopped => { stream_context = None; diff --git a/alvr/client_openxr/src/lobby.rs b/alvr/client_openxr/src/lobby.rs index 49634aadab..aa223fe760 100644 --- a/alvr/client_openxr/src/lobby.rs +++ b/alvr/client_openxr/src/lobby.rs @@ -129,6 +129,20 @@ impl Lobby { &mut Vec3::new(0.0, 0.0, 0.0), ); + let body_skeleton_fb = self + .interaction_ctx + .body_sources + .body_tracker_fb + .as_ref() + .and_then(|(tracker, joint_count)| { + interaction::get_fb_body_skeleton( + &self.reference_space, + predicted_display_time, + tracker, + *joint_count, + ) + }); + let left_swapchain_idx = self.swapchains[0].acquire_image().unwrap(); let right_swapchain_idx = self.swapchains[1].acquire_image().unwrap(); @@ -156,6 +170,7 @@ impl Lobby { (left_hand_data.0.map(|dm| dm.pose), left_hand_data.1), (right_hand_data.0.map(|dm| dm.pose), right_hand_data.1), ], + body_skeleton_fb, ); self.swapchains[0].release_image().unwrap(); diff --git a/alvr/client_openxr/src/stream.rs b/alvr/client_openxr/src/stream.rs index cb7f9b9e2e..037bf88a5b 100644 --- a/alvr/client_openxr/src/stream.rs +++ b/alvr/client_openxr/src/stream.rs @@ -94,33 +94,6 @@ impl StreamContext { .request_display_refresh_rate(config.refresh_rate_hint) .unwrap(); } - // todo: check which permissions are needed for htc - #[cfg(target_os = "android")] - if let Some(config) = &config.face_sources_config { - if (config.combined_eye_gaze || config.eye_tracking_fb) - && matches!(platform, Platform::Quest3 | Platform::QuestPro) - { - alvr_client_core::try_get_permission("com.oculus.permission.EYE_TRACKING") - } - if config.combined_eye_gaze && matches!(platform, Platform::Pico4 | Platform::PicoNeo3) - { - alvr_client_core::try_get_permission("com.picovr.permission.EYE_TRACKING") - } - if config.face_tracking_fb && matches!(platform, Platform::Quest3 | Platform::QuestPro) - { - alvr_client_core::try_get_permission("android.permission.RECORD_AUDIO"); - alvr_client_core::try_get_permission("com.oculus.permission.FACE_TRACKING") - } - } - - #[cfg(target_os = "android")] - if let Some(config) = &config.body_sources_config { - if (config.body_tracking_full_body_meta.enabled()) - && matches!(platform, Platform::Quest3 | Platform::QuestPro) - { - alvr_client_core::try_get_permission("com.oculus.permission.BODY_TRACKING") - } - } let foveation_profile = if let Some(config) = &config.clientside_foveation_config { if xr_ctx.instance.exts().fb_swapchain_update_state.is_some() @@ -487,14 +460,12 @@ fn stream_input_loop( htc_lip_expression: interaction::get_htc_lip_expression(&interaction_ctx.face_sources), }; - if let Some(body_tracker_full_body_meta) = - &interaction_ctx.body_sources.body_tracker_full_body_meta - { - device_motions.append(&mut interaction::get_meta_body_tracking_full_body_points( + if let Some((tracker, joint_count)) = &interaction_ctx.body_sources.body_tracker_fb { + device_motions.append(&mut interaction::get_fb_body_tracking_points( &reference_space, crate::to_xr_time(now), - body_tracker_full_body_meta, - interaction_ctx.body_sources.enable_full_body, + tracker, + *joint_count, )); } diff --git a/alvr/server_core/src/connection.rs b/alvr/server_core/src/connection.rs index a173df3999..07260a18d8 100644 --- a/alvr/server_core/src/connection.rs +++ b/alvr/server_core/src/connection.rs @@ -104,16 +104,13 @@ pub fn contruct_openvr_config(session: &SessionConfig) -> OpenvrConfig { }; // Should be true if using full body tracking - let body_tracking_has_legs = if let Switch::Enabled(config) = &settings.headset.body_tracking { - if let Switch::Enabled(body_source_settings) = &config.sources.body_tracking_full_body_meta - { - body_source_settings.enable_full_body - } else { - false - } - } else { - false - }; + let body_tracking_has_legs = settings + .headset + .body_tracking + .as_option() + .and_then(|c| c.sources.body_tracking_fb.as_option().cloned()) + .map(|c| c.full_body) + .unwrap_or(false); let mut foveation_center_size_x = 0.0; let mut foveation_center_size_y = 0.0; diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 6258aea6d2..de439abc3a 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -682,14 +682,12 @@ pub struct FaceTrackingConfig { #[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq)] pub struct BodyTrackingSourcesConfig { - pub body_tracking_full_body_meta: Switch, + pub body_tracking_fb: Switch, } #[derive(SettingsSchema, Serialize, Deserialize, Clone, PartialEq)] -#[schema(collapsible)] -pub struct BodyTrackingFullBodyMETAConfig { - #[schema(strings(help = "Enable full body tracking"))] - pub enable_full_body: bool, +pub struct BodyTrackingFBConfig { + pub full_body: bool, } #[derive(SettingsSchema, Serialize, Deserialize, Clone)] @@ -1514,12 +1512,9 @@ pub fn session_settings_default() -> SettingsDefault { content: BodyTrackingConfigDefault { gui_collapsed: true, sources: BodyTrackingSourcesConfigDefault { - body_tracking_full_body_meta: SwitchDefault { + body_tracking_fb: SwitchDefault { enabled: true, - content: BodyTrackingFullBodyMETAConfigDefault { - gui_collapsed: true, - enable_full_body: true, - }, + content: BodyTrackingFBConfigDefault { full_body: true }, }, }, sink: BodyTrackingSinkConfigDefault {