kohrokho
/
ALVR-IPv6
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Send negotiated config as JSON, apply client-side foveation (#1646)

This commit is contained in:
Riccardo Zaglia 2023-05-30 18:40:33 +08:00 committed by GitHub
parent 085c3f53d2
commit 27c86ca874
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 179 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Cargo.lock generated
View File

@ -157,7 +157,7 @@ dependencies = [
[[package]]
name = "alvr_audio"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"alvr_session",
@ -172,7 +172,7 @@ dependencies = [
[[package]]
name = "alvr_client_core"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_audio",
"alvr_common",
@ -203,7 +203,7 @@ dependencies = [
[[package]]
name = "alvr_client_mock"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_client_core",
"alvr_common",
@ -216,7 +216,7 @@ dependencies = [
[[package]]
name = "alvr_client_openxr"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_client_core",
"alvr_common",
@ -233,7 +233,7 @@ dependencies = [
[[package]]
name = "alvr_common"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"backtrace",
"glam",
@ -248,7 +248,7 @@ dependencies = [
[[package]]
name = "alvr_dashboard"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"alvr_events",
@ -273,7 +273,7 @@ dependencies = [
[[package]]
name = "alvr_events"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"alvr_packets",
@ -284,7 +284,7 @@ dependencies = [
[[package]]
name = "alvr_filesystem"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"dirs 5.0.1",
"once_cell",
@ -292,7 +292,7 @@ dependencies = [
[[package]]
name = "alvr_packets"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"alvr_session",
@ -302,7 +302,7 @@ dependencies = [
[[package]]
name = "alvr_server"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_audio",
"alvr_common",
@ -336,7 +336,7 @@ dependencies = [
[[package]]
name = "alvr_server_io"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"alvr_events",
@ -352,7 +352,7 @@ dependencies = [
[[package]]
name = "alvr_session"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"bytemuck",
@ -364,7 +364,7 @@ dependencies = [
[[package]]
name = "alvr_sockets"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_common",
"alvr_session",
@ -380,14 +380,14 @@ dependencies = [
[[package]]
name = "alvr_vrcompositor_wrapper"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"exec",
]
[[package]]
name = "alvr_vulkan_layer"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_filesystem",
"bindgen 0.65.1",
@ -398,7 +398,7 @@ dependencies = [
[[package]]
name = "alvr_xtask"
version = "20.0.0-dev14"
version = "20.0.0-dev15"
dependencies = [
"alvr_filesystem",
"pico-args",
@ -3297,7 +3297,7 @@ dependencies = [
[[package]]
name = "openxr"
version = "0.17.1"
source = "git+https://github.com/zarik5/openxrs#144c8d4bf4e63741e20c447e00ec46428fae2697"
source = "git+https://github.com/zarik5/openxrs#2fe331ec5f97f709e8126061503f8922b434bc09"
dependencies = [
"libc",
"libloading 0.7.4",
@ -3308,7 +3308,7 @@ dependencies = [
[[package]]
name = "openxr-sys"
version = "0.9.3"
source = "git+https://github.com/zarik5/openxrs#144c8d4bf4e63741e20c447e00ec46428fae2697"
source = "git+https://github.com/zarik5/openxrs#2fe331ec5f97f709e8126061503f8922b434bc09"
dependencies = [
"libc",
]

2
Cargo.toml
View File

@ -3,7 +3,7 @@ resolver = "2"
members = ["alvr/*"]
[workspace.package]
version = "20.0.0-dev14"
version = "20.0.0-dev15"
edition = "2021"
rust-version = "1.65"
authors = ["alvr-org"]

29
alvr/client_core/src/connection.rs
View File

@ -23,6 +23,7 @@ use alvr_sockets::{
use futures::future::BoxFuture;
use serde_json as json;
use std::{
collections::HashMap,
future,
net::IpAddr,
sync::Arc,
@ -193,14 +194,30 @@ async fn stream_pipeline(
) -> StrResult {
let settings = {
let mut session_desc = SessionDesc::default();
session_desc
.merge_from_json(&json::from_str(&stream_config.session_desc).map_err(err!())?)?;
session_desc.merge_from_json(&json::from_str(&stream_config.session).map_err(err!())?)?;
session_desc.to_settings()
};
let negotiated_config =
json::from_str::<HashMap<String, json::Value>>(&stream_config.negotiated)
.map_err(err!())?;
let view_resolution = negotiated_config
.get("view_resolution")
.and_then(|v| json::from_value(v.clone()).ok())
.unwrap_or(UVec2::ZERO);
let refresh_rate_hint = negotiated_config
.get("refresh_rate_hint")
.and_then(|v| v.as_f64())
.unwrap_or(60.0) as f32;
let game_audio_sample_rate = negotiated_config
.get("game_audio_sample_rate")
.and_then(|v| v.as_u64())
.unwrap_or(44100) as u32;
let streaming_start_event = ClientCoreEvent::StreamingStarted {
view_resolution: stream_config.view_resolution,
refresh_rate_hint: stream_config.fps,
view_resolution,
refresh_rate_hint,
settings: Box::new(settings.clone()),
};
@ -231,7 +248,7 @@ async fn stream_pipeline(
*STATISTICS_MANAGER.lock() = Some(StatisticsManager::new(
settings.connection.statistics_history_size as _,
Duration::from_secs_f32(1.0 / stream_config.fps),
Duration::from_secs_f32(1.0 / refresh_rate_hint),
if let Switch::Enabled(config) = settings.headset.controllers {
config.steamvr_pipeline_frames
} else {
@ -417,7 +434,7 @@ async fn stream_pipeline(
Box::pin(audio::play_audio_loop(
device,
2,
stream_config.game_audio_sample_rate,
game_audio_sample_rate,
config.buffering,
game_audio_receiver,
))

104
alvr/client_openxr/src/lib.rs
View File

@ -8,6 +8,7 @@ use alvr_common::{
DeviceMotion, Fov, Pose, RelaxedAtomic, HEAD_ID, LEFT_HAND_ID, RIGHT_HAND_ID,
};
use alvr_packets::{FaceData, Tracking};
use alvr_session::ClientsideFoveationMode;
use interaction::{FaceInputContext, HandsInteractionContext};
use khronos_egl::{self as egl, EGL1_4};
use openxr as xr;
@ -19,6 +20,7 @@ use std::{
thread,
time::{Duration, Instant},
};
use xr::FoveationProfileFB;
const IPD_CHANGE_EPS: f32 = 0.001;
const DECODER_MAX_TIMEOUT_MULTIPLIER: f32 = 0.8;
@ -184,21 +186,34 @@ fn create_xr_session(
pub fn create_swapchain(
session: &xr::Session<xr::OpenGlEs>,
resolution: UVec2,
foveation: Option<&FoveationProfileFB>,
) -> xr::Swapchain<xr::OpenGlEs> {
session
.create_swapchain(&xr::SwapchainCreateInfo {
create_flags: xr::SwapchainCreateFlags::EMPTY,
usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT
| xr::SwapchainUsageFlags::SAMPLED,
format: glow::SRGB8_ALPHA8,
sample_count: 1,
width: resolution.x,
height: resolution.y,
face_count: 1,
array_size: 1,
mip_count: 1,
})
.unwrap()
let swapchain_info = xr::SwapchainCreateInfo {
create_flags: xr::SwapchainCreateFlags::EMPTY,
usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT | xr::SwapchainUsageFlags::SAMPLED,
format: glow::SRGB8_ALPHA8,
sample_count: 1,
width: resolution.x,
height: resolution.y,
face_count: 1,
array_size: 1,
mip_count: 1,
};
if let Some(foveation) = foveation {
let swapchain = session
.create_swapchain_with_foveation(
&swapchain_info,
xr::SwapchainCreateFoveationFlagsFB::SCALED_BIN,
)
.unwrap();
swapchain.update_foveation(foveation).unwrap();
swapchain
} else {
session.create_swapchain(&swapchain_info).unwrap()
}
}
// This function is allowed to return errors. It can happen when the session is destroyed
@ -365,6 +380,9 @@ pub fn entry_point() {
exts.fb_display_refresh_rate = available_extensions.fb_display_refresh_rate;
exts.fb_eye_tracking_social = available_extensions.fb_eye_tracking_social;
exts.fb_face_tracking = available_extensions.fb_face_tracking;
exts.fb_foveation = available_extensions.fb_foveation;
exts.fb_foveation_configuration = available_extensions.fb_foveation_configuration;
exts.fb_swapchain_update_state = available_extensions.fb_swapchain_update_state;
exts.htc_facial_tracking = available_extensions.htc_facial_tracking;
exts.htc_vive_focus3_controller_interaction =
available_extensions.htc_vive_focus3_controller_interaction;
@ -484,8 +502,16 @@ pub fn entry_point() {
let swapchains = lobby_swapchains.get_or_insert_with(|| {
[
create_swapchain(&xr_session, recommended_view_resolution),
create_swapchain(&xr_session, recommended_view_resolution),
create_swapchain(
&xr_session,
recommended_view_resolution,
None,
),
create_swapchain(
&xr_session,
recommended_view_resolution,
None,
),
]
});
@ -684,10 +710,52 @@ pub fn entry_point() {
}
}));
let foveation_profile = if let Some(config) =
settings.video.clientside_foveation.into_option()
{
if exts.fb_swapchain_update_state
&& exts.fb_foveation
&& exts.fb_foveation_configuration
{
let level;
let dynamic;
match config.mode {
ClientsideFoveationMode::Static { level: lvl } => {
level = lvl;
dynamic = false;
}
ClientsideFoveationMode::Dynamic { max_level } => {
level = max_level;
dynamic = true;
}
};
xr_session
.create_foveation_profile(Some(xr::FoveationLevelProfile {
level: xr::FoveationLevelFB::from_raw(level as i32),
vertical_offset: config.vertical_offset_deg,
dynamic: xr::FoveationDynamicFB::from_raw(dynamic as i32),
}))
.ok()
} else {
None
}
} else {
None
};
let swapchains = stream_swapchains.get_or_insert_with(|| {
[
create_swapchain(&xr_session, stream_view_resolution),
create_swapchain(&xr_session, stream_view_resolution),
create_swapchain(
&xr_session,
stream_view_resolution,
foveation_profile.as_ref(),
),
create_swapchain(
&xr_session,
stream_view_resolution,
foveation_profile.as_ref(),
),
]
});

7
alvr/packets/src/lib.rs
View File

@ -37,10 +37,9 @@ pub enum ClientConnectionResult {
#[derive(Serialize, Deserialize)]
pub struct StreamConfigPacket {
pub session_desc: String, // transfer session as string to allow for extrapolation
pub view_resolution: UVec2,
pub fps: f32,
pub game_audio_sample_rate: u32,
pub session: String, // JSON session that allows for extrapolation
pub negotiated: String, // JSON dictionary containing negotiated configuration. Can be extended
// without a breaking protocol change, but entries can't be removed.
}
#[derive(Serialize, Deserialize, Clone)]

11
alvr/server/src/connection.rs
View File

@ -278,13 +278,16 @@ fn try_connect(mut client_ips: HashMap<IpAddr, String>) -> IntResult {
};
let client_config = StreamConfigPacket {
session_desc: {
session: {
let session = SERVER_DATA_MANAGER.read().session().clone();
serde_json::to_string(&session).map_err(to_int_e!())?
},
view_resolution: stream_view_resolution,
fps,
game_audio_sample_rate,
negotiated: serde_json::json!({
"view_resolution": stream_view_resolution,
"refresh_rate_hint": fps,
"game_audio_sample_rate": game_audio_sample_rate,
})
.to_string(),
};
runtime
.block_on(proto_socket.send(&client_config))

47
alvr/session/src/settings.rs
View File

@ -297,12 +297,25 @@ pub struct BitrateConfig {
#[repr(u8)]
#[derive(SettingsSchema, Serialize, Deserialize, Copy, Clone)]
pub enum OculusFovetionLevel {
None,
Low,
Medium,
High,
HighTop,
pub enum ClientsideFoveationLevel {
Low = 1,
Medium = 2,
High = 3,
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
pub enum ClientsideFoveationMode {
Static { level: ClientsideFoveationLevel },
Dynamic { max_level: ClientsideFoveationLevel },
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
pub struct ClientsideFoveation {
pub mode: ClientsideFoveationMode,
#[schema(strings(display_name = "Foveation offset"))]
#[schema(gui(slider(min = -45.0, max = 45.0, step = 0.1)), suffix = "°")]
pub vertical_offset_deg: f32,
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
@ -425,7 +438,7 @@ pub struct VideoDesc {
#[schema(flag = "steamvr-restart")]
pub foveated_rendering: Switch<FoveatedRenderingDesc>,
pub oculus_foveation_level: OculusFovetionLevel,
pub clientside_foveation: Switch<ClientsideFoveation>,
pub dynamic_oculus_foveation: bool,
@ -1025,8 +1038,24 @@ pub fn session_settings_default() -> SettingsDefault {
edge_ratio_y: 5.,
},
},
oculus_foveation_level: OculusFovetionLevelDefault {
variant: OculusFovetionLevelDefaultVariant::HighTop,
clientside_foveation: SwitchDefault {
enabled: true,
content: ClientsideFoveationDefault {
mode: ClientsideFoveationModeDefault {
Static: ClientsideFoveationModeStaticDefault {
level: ClientsideFoveationLevelDefault {
variant: ClientsideFoveationLevelDefaultVariant::High,
},
},
Dynamic: ClientsideFoveationModeDynamicDefault {
max_level: ClientsideFoveationLevelDefault {
variant: ClientsideFoveationLevelDefaultVariant::High,
},
},
variant: ClientsideFoveationModeDefaultVariant::Dynamic,
},
vertical_offset_deg: 0.0,
},
},
dynamic_oculus_foveation: true,
color_correction: SwitchDefault {

2
packaging/deb/control
View File

@ -34,7 +34,7 @@ Vcs-Browser: https://github.com/alvr-org/ALVR
Vcs-Git: https://github.com/alvr-org/ALVR.git
Rules-Requires-Root: no
Package: alvr
Version: 20.0.0-dev14
Version: 20.0.0-dev15
Architecture: amd64
Recommends: steam
Description: Stream VR games from your PC to your headset via Wi-Fi

4
packaging/rpm/alvr.spec
View File

@ -1,9 +1,9 @@
Name: alvr
Version: 20.0.0
Release: 0.0.1dev14
Release: 0.0.1dev15
Summary: Stream VR games from your PC to your headset via Wi-Fi
License: MIT
Source: https://github.com/alvr-org/ALVR/archive/refs/tags/v20.0.0-dev14.tar.gz
Source: https://github.com/alvr-org/ALVR/archive/refs/tags/v20.0.0-dev15.tar.gz
URL: https://github.com/alvr-org/ALVR/
ExclusiveArch: x86_64
BuildRequires: alsa-lib-devel cairo-gobject-devel cargo clang-devel ffmpeg-devel gcc gcc-c++ cmake ImageMagick (jack-audio-connection-kit-devel or pipewire-jack-audio-connection-kit-devel) libunwind-devel openssl-devel rpmdevtools rust rust-atk-sys-devel rust-cairo-sys-rs-devel rust-gdk-sys-devel rust-glib-sys-devel rust-pango-sys-devel selinux-policy-devel vulkan-headers vulkan-loader-devel