Refactor bitrate options

Apply some options only when adaptive bitrate is selected
Allow to disable framerate normalization
This commit is contained in:
Riccardo Zaglia 2023-04-12 22:31:14 +02:00
parent 4fa46ce701
commit 528fe67df4
5 changed files with 158 additions and 98 deletions

88
Cargo.lock generated
View File

@ -401,7 +401,7 @@ dependencies = [
"futures",
"serde",
"serde_json",
"socket2 0.5.1",
"socket2 0.5.2",
"tokio",
"tokio-util",
]
@ -652,7 +652,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -669,7 +669,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -756,7 +756,7 @@ dependencies = [
"cc",
"cfg-if",
"libc",
"miniz_oxide",
"miniz_oxide 0.6.2",
"object",
"rustc-demangle",
]
@ -897,7 +897,7 @@ checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -1276,7 +1276,7 @@ dependencies = [
"proc-macro2",
"quote",
"scratch",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -1293,7 +1293,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -1599,7 +1599,7 @@ checksum = "9d2c772ccdbdfd1967b4f5d79d17c98ebf92009fdcc838db7aa434462f600c26"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -1698,6 +1698,15 @@ dependencies = [
"instant",
]
[[package]]
name = "fdeflate"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10"
dependencies = [
"simd-adler32",
]
[[package]]
name = "fern"
version = "0.6.2"
@ -1724,7 +1733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
dependencies = [
"crc32fast",
"miniz_oxide",
"miniz_oxide 0.6.2",
]
[[package]]
@ -1828,7 +1837,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -2855,6 +2864,16 @@ dependencies = [
"adler",
]
[[package]]
name = "miniz_oxide"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
dependencies = [
"adler",
"simd-adler32",
]
[[package]]
name = "mio"
version = "0.8.6"
@ -3260,7 +3279,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -3438,14 +3457,15 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "png"
version = "0.17.7"
version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d708eaf860a19b19ce538740d2b4bdeeb8337fa53f7738455e706623ad5c638"
checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa"
dependencies = [
"bitflags",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
"miniz_oxide 0.7.1",
]
[[package]]
@ -3849,9 +3869,9 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.159"
version = "1.0.160"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
dependencies = [
"serde_derive",
]
@ -3870,13 +3890,13 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.159"
version = "1.0.160"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -3898,7 +3918,7 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -3925,7 +3945,7 @@ dependencies = [
[[package]]
name = "settings-schema"
version = "0.2.0"
source = "git+https://github.com/zarik5/settings-schema-rs#be3dbb9c923cdacb5e1970687bd6e943c55448ec"
source = "git+https://github.com/zarik5/settings-schema-rs#5f062a058af370a3dc6603f2a0b6401eaf29ec87"
dependencies = [
"serde",
"serde_json",
@ -3935,7 +3955,7 @@ dependencies = [
[[package]]
name = "settings-schema-derive"
version = "0.2.0"
source = "git+https://github.com/zarik5/settings-schema-rs#be3dbb9c923cdacb5e1970687bd6e943c55448ec"
source = "git+https://github.com/zarik5/settings-schema-rs#5f062a058af370a3dc6603f2a0b6401eaf29ec87"
dependencies = [
"darling",
"proc-macro2",
@ -3969,6 +3989,12 @@ dependencies = [
"libc",
]
[[package]]
name = "simd-adler32"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f"
[[package]]
name = "slab"
version = "0.4.8"
@ -4034,12 +4060,12 @@ dependencies = [
[[package]]
name = "socket2"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc8d618c6641ae355025c449427f9e96b98abf99a772be3cef6708d15c77147a"
checksum = "6d283f86695ae989d1e18440a943880967156325ba025f05049946bff47bcc2b"
dependencies = [
"libc",
"windows-sys 0.45.0",
"windows-sys 0.48.0",
]
[[package]]
@ -4155,9 +4181,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.13"
version = "2.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5"
dependencies = [
"proc-macro2",
"quote",
@ -4230,7 +4256,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -4320,7 +4346,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@ -4795,9 +4821,9 @@ dependencies = [
[[package]]
name = "webbrowser"
version = "0.8.8"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "579cc485bd5ce5bfa0d738e4921dd0b956eca9800be1fd2e5257ebe95bc4617e"
checksum = "b692165700260bbd40fbc5ff23766c03e339fbaca907aeea5cb77bf0a553ca83"
dependencies = [
"core-foundation",
"dirs 4.0.0",

View File

@ -10,6 +10,7 @@ const UPDATE_INTERVAL: Duration = Duration::from_secs(1);
pub struct BitrateManager {
config: BitrateConfig,
nominal_framerate: f32,
max_history_size: usize,
frame_interval_average: SlidingWindowAverage<Duration>,
packet_sizes_bits_history: VecDeque<(Duration, usize)>,
@ -23,9 +24,10 @@ pub struct BitrateManager {
}
impl BitrateManager {
pub fn new(config: BitrateConfig, max_history_size: usize) -> Self {
pub fn new(config: BitrateConfig, max_history_size: usize, nominal_framerate: f32) -> Self {
Self {
config,
nominal_framerate,
max_history_size,
frame_interval_average: SlidingWindowAverage::new(
Duration::from_millis(16),
@ -47,22 +49,25 @@ impl BitrateManager {
// Note: This is used to calculate the framerate/frame interval. The frame present is the most
// accurate event for this use.
pub fn report_frame_resent(&mut self) {
pub fn report_frame_present(&mut self) {
let now = Instant::now();
let interval = now - self.last_frame_instant;
self.last_frame_instant = now;
// If the latest frame interval deviates too much from the mean,
let interval_ratio =
interval.as_secs_f32() / self.frame_interval_average.get_average().as_secs_f32();
if let Switch::Enabled(config) = &self.config.adapt_to_framerate {
let interval_ratio =
interval.as_secs_f32() / self.frame_interval_average.get_average().as_secs_f32();
self.frame_interval_average.submit_sample(interval);
self.frame_interval_average.submit_sample(interval);
if (interval_ratio - 1.0).abs() > self.config.framerate_reset_threshold_multiplier {
self.frame_interval_average =
SlidingWindowAverage::new(interval, self.max_history_size);
self.update_needed = true;
if interval_ratio > config.framerate_reset_threshold_multiplier
|| interval_ratio < 1.0 / config.framerate_reset_threshold_multiplier
{
self.frame_interval_average =
SlidingWindowAverage::new(interval, self.max_history_size);
self.update_needed = true;
}
}
}
@ -96,7 +101,11 @@ impl BitrateManager {
}
}
if let Switch::Enabled(config) = &self.config.decoder_latency_fixer {
if let BitrateMode::Adaptive {
decoder_latency_fixer: Switch::Enabled(config),
..
} = &self.config.mode
{
if decoder_latency > Duration::from_millis(config.max_decoder_latency_ms) {
self.decoder_latency_overstep_count += 1;
@ -127,12 +136,14 @@ impl BitrateManager {
};
}
let mut bitrate_bps = match &self.config.mode {
let bitrate_bps = match &self.config.mode {
BitrateMode::ConstantMbps(bitrate_mbps) => *bitrate_mbps as f32 * 1e6,
BitrateMode::Adaptive {
saturation_multiplier,
max_bitrate_mbps,
min_bitrate_mbps,
max_network_latency_ms,
..
} => {
let mut bitrate_bps = self.bitrate_average.get_average() * saturation_multiplier;
@ -143,24 +154,28 @@ impl BitrateManager {
bitrate_bps = f32::max(bitrate_bps, *min as f32 * 1e6);
}
if let Switch::Enabled(max_ms) = max_network_latency_ms {
let multiplier = *max_ms as f32
/ 1000.0
/ self.network_latency_average.get_average().as_secs_f32();
bitrate_bps = f32::min(bitrate_bps, bitrate_bps * multiplier);
}
bitrate_bps = f32::min(bitrate_bps, self.dynamic_max_bitrate);
bitrate_bps
}
};
if let Switch::Enabled(max_ms) = &self.config.max_network_latency_ms {
let multiplier =
*max_ms as f32 / 1000.0 / self.network_latency_average.get_average().as_secs_f32();
bitrate_bps = f32::min(bitrate_bps, bitrate_bps * multiplier);
}
bitrate_bps = f32::min(bitrate_bps, self.dynamic_max_bitrate);
let framerate = 1.0
/ self
let framerate = if self.config.adapt_to_framerate.enabled() {
1.0 / self
.frame_interval_average
.get_average()
.as_secs_f32()
.min(1.0);
.min(1.0)
} else {
self.nominal_framerate
};
FfiDynamicEncoderParams {
updated: 1,

View File

@ -565,6 +565,7 @@ async fn connection_pipeline(
*BITRATE_MANAGER.lock() = BitrateManager::new(
settings.video.bitrate,
settings.connection.statistics_history_size as _,
refresh_rate,
);
// todo: dynamic framerate

View File

@ -66,9 +66,11 @@ static WEBSERVER_RUNTIME: Lazy<Mutex<Option<Runtime>>> =
static STATISTICS_MANAGER: Lazy<Mutex<Option<StatisticsManager>>> = Lazy::new(|| Mutex::new(None));
static BITRATE_MANAGER: Lazy<Mutex<BitrateManager>> = Lazy::new(|| {
let data_lock = SERVER_DATA_MANAGER.read();
let settings = data_lock.settings();
Mutex::new(BitrateManager::new(
data_lock.settings().video.bitrate.clone(),
data_lock.settings().connection.statistics_history_size as usize,
settings.video.bitrate.clone(),
settings.connection.statistics_history_size as usize,
settings.video.preferred_fps,
))
});
@ -428,7 +430,7 @@ pub unsafe extern "C" fn HmdDriverFactory(
);
}
BITRATE_MANAGER.lock().report_frame_resent();
BITRATE_MANAGER.lock().report_frame_present();
}
extern "C" fn report_composed(timestamp_ns: u64, offset_ns: u64) {

View File

@ -211,28 +211,6 @@ pub enum MediacodecDataType {
String(String),
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
#[schema(gui = "button_group")]
pub enum BitrateMode {
#[schema(strings(display_name = "Constant"))]
ConstantMbps(#[schema(gui(slider(min = 5, max = 1000, logarithmic)), suffix = "Mbps")] u64),
Adaptive {
#[schema(strings(
help = "Percentage of network bandwidth to allocate for video transmission"
))]
#[schema(gui(slider(min = 0.5, max = 2.0, step = 0.05)))]
saturation_multiplier: f32,
#[schema(strings(display_name = "Maximum bitrate"))]
#[schema(gui(slider(min = 1, max = 1000, logarithmic)), suffix = "Mbps")]
max_bitrate_mbps: Switch<u64>,
#[schema(strings(display_name = "Minimum bitrate"))]
#[schema(gui(slider(min = 1, max = 1000, logarithmic)), suffix = "Mbps")]
min_bitrate_mbps: Switch<u64>,
},
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
pub struct DecoderLatencyFixer {
#[schema(strings(
@ -256,21 +234,54 @@ pub struct DecoderLatencyFixer {
pub latency_overstep_multiplier: f32,
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
#[schema(gui = "button_group")]
pub enum BitrateMode {
#[schema(strings(display_name = "Constant"))]
ConstantMbps(#[schema(gui(slider(min = 5, max = 1000, logarithmic)), suffix = "Mbps")] u64),
Adaptive {
#[schema(strings(
help = "Percentage of network bandwidth to allocate for video transmission"
))]
#[schema(gui(slider(min = 0.5, max = 2.0, step = 0.05)))]
saturation_multiplier: f32,
#[schema(strings(display_name = "Maximum bitrate"))]
#[schema(gui(slider(min = 1, max = 1000, logarithmic)), suffix = "Mbps")]
max_bitrate_mbps: Switch<u64>,
#[schema(strings(display_name = "Minimum bitrate"))]
#[schema(gui(slider(min = 1, max = 1000, logarithmic)), suffix = "Mbps")]
min_bitrate_mbps: Switch<u64>,
#[schema(strings(display_name = "Maximum network latency"))]
#[schema(gui(slider(min = 1, max = 50)), suffix = "ms")]
max_network_latency_ms: Switch<u64>,
#[schema(strings(
help = "Currently there is a bug where the decoder latency keeps rising when above a certain bitrate"
))]
decoder_latency_fixer: Switch<DecoderLatencyFixer>,
},
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
pub struct BitrateAdaptiveFramerateConfig {
#[schema(strings(
help = "If the framerate changes more than this factor, trigger a parameters update"
))]
#[schema(gui(slider(min = 1.0, max = 3.0, step = 0.1)))]
pub framerate_reset_threshold_multiplier: f32,
}
#[derive(SettingsSchema, Serialize, Deserialize, Clone)]
pub struct BitrateConfig {
pub mode: BitrateMode,
#[schema(gui(slider(min = 0.01, max = 2.0, step = 0.01)))]
pub framerate_reset_threshold_multiplier: f32,
#[schema(strings(display_name = "Maximum network latency"))]
#[schema(gui(slider(min = 1, max = 50)), suffix = "ms")]
pub max_network_latency_ms: Switch<u64>,
#[schema(strings(
help = "Currently there is a bug where the decoder latency keeps rising when above a certain bitrate"
help = "Ensure that the specified bitrate value is respected regardless of the framerate"
))]
pub decoder_latency_fixer: Switch<DecoderLatencyFixer>,
pub adapt_to_framerate: Switch<BitrateAdaptiveFramerateConfig>,
}
#[repr(u8)]
@ -843,20 +854,25 @@ pub fn session_settings_default() -> SettingsDefault {
enabled: false,
content: 5,
},
max_network_latency_ms: SwitchDefault {
enabled: false,
content: 8,
},
decoder_latency_fixer: SwitchDefault {
enabled: true,
content: DecoderLatencyFixerDefault {
max_decoder_latency_ms: 20,
latency_overstep_frames: 30,
latency_overstep_multiplier: 0.99,
},
},
},
variant: BitrateModeDefaultVariant::Adaptive,
},
framerate_reset_threshold_multiplier: 0.30,
max_network_latency_ms: SwitchDefault {
enabled: false,
content: 8,
},
decoder_latency_fixer: SwitchDefault {
adapt_to_framerate: SwitchDefault {
enabled: true,
content: DecoderLatencyFixerDefault {
max_decoder_latency_ms: 20,
latency_overstep_frames: 30,
latency_overstep_multiplier: 0.99,
content: BitrateAdaptiveFramerateConfigDefault {
framerate_reset_threshold_multiplier: 2.0,
},
},
},