Skip to content

Commit

Permalink
Android: When the hardware/default decoder fails to initialize, fall …
Browse files Browse the repository at this point in the history 
…back to software. (#1933)

* Android: When the hardware/default decoder fails to initialize, fall back to software.

* Forgot to "cargo fmt --all", thank you CI

* Android: Reduce unwraps in decoder setup to one unified place so that can be figured out later

* Android: decoder.rs: cargo fmt --all

* Add option to force software decoding

* Android: Reduce use of unwrapping in decoder setup. Logging: errors should be more explicit

* Settings: Move force_software_decoder above mediacodec_extra_options, Logging: cargo fmt --all

* Revert logging changes made for Android decoder error messages

* Android: Use match instead of if-let for decoder error handling
  • Loading branch information
@20kdc
20kdc authored Jan 3, 2024
1 parent 12e6ac6 commit 377c556
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 61 deletions.
2 changes: 1 addition & 1 deletion alvr/client_core/src/connection.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ fn connection_pipeline(

match maybe_packet {
Ok(ServerControlPacket::InitializeDecoder(config)) => {
decoder::create_decoder(config);
decoder::create_decoder(config, settings.video.force_software_decoder);
}
Ok(ServerControlPacket::Restarting) => {
info!("{SERVER_RESTART_MESSAGE}");
Expand Down
5 changes: 4 additions & 1 deletion alvr/client_core/src/decoder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use std::time::Duration;
#[derive(Clone)]
pub struct DecoderInitConfig {
pub codec: CodecType,
pub force_software_decoder: bool,
pub max_buffering_frames: f32,
pub buffering_history_weight: f32,
pub options: Vec<(String, MediacodecDataType)>,
Expand All @@ -15,6 +16,7 @@ pub struct DecoderInitConfig {
pub static DECODER_INIT_CONFIG: Lazy<Mutex<DecoderInitConfig>> = Lazy::new(|| {
Mutex::new(DecoderInitConfig {
codec: CodecType::H264,
force_software_decoder: false,
max_buffering_frames: 1.0,
buffering_history_weight: 0.9,
options: vec![],
Expand All @@ -29,9 +31,10 @@ pub static DECODER_SOURCE: alvr_common::OptLazy<crate::platform::VideoDecoderSou

pub static EXTERNAL_DECODER: RelaxedAtomic = RelaxedAtomic::new(false);

pub fn create_decoder(lazy_config: DecoderInitializationConfig) {
pub fn create_decoder(lazy_config: DecoderInitializationConfig, force_software_decoder: bool) {
let mut config = DECODER_INIT_CONFIG.lock();
config.codec = lazy_config.codec;
config.force_software_decoder = force_software_decoder;

if EXTERNAL_DECODER.value() {
EVENT_QUEUE
Expand Down
171 changes: 112 additions & 59 deletions alvr/client_core/src/platform/android/decoder.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use crate::decoder::DecoderInitConfig;
use alvr_common::{
anyhow::{bail, Result},
anyhow::{anyhow, bail, Context, Result},
error, info,
parking_lot::{Condvar, Mutex},
warn, RelaxedAtomic,
show_e, warn, RelaxedAtomic,
};
use alvr_session::{CodecType, MediacodecDataType};
use ndk::{
Expand Down Expand Up @@ -144,6 +144,43 @@ impl Drop for VideoDecoderSource {
}
}

fn mime_for_codec(codec: CodecType) -> &'static str {
match codec {
CodecType::H264 => "video/avc",
CodecType::Hevc => "video/hevc",
}
}

// Attempts to create a MediaCodec, and then configure and start it.
fn decoder_attempt_setup(
codec_type: CodecType,
is_software: bool,
format: &MediaFormat,
image_reader: &ImageReader,
) -> Result<MediaCodec> {
let decoder = if is_software {
let sw_codec_name = match codec_type {
CodecType::H264 => "OMX.google.h264.decoder",
CodecType::Hevc => "OMX.google.hevc.decoder",
};
MediaCodec::from_codec_name(&sw_codec_name)
.ok_or(anyhow!("no such codec: {}", &sw_codec_name))?
} else {
let mime = mime_for_codec(codec_type);
MediaCodec::from_decoder_type(&mime)
.ok_or(anyhow!("unable to find decoder for mime type: {}", &mime))?
};
let decoder_configure_err = decoder.configure(
&format,
Some(&image_reader.window()?),
MediaCodecDirection::Decoder,
);
decoder_configure_err.with_context(|| format!("failed to configure decoder"))?;
let decoder_start_err = decoder.start();
decoder_start_err.with_context(|| format!("failed to start decoder"))?;
Ok(decoder)
}

// Create a sink/source pair
pub fn video_decoder_split(
config: DecoderInitConfig,
Expand All @@ -167,26 +204,6 @@ pub fn video_decoder_split(
// 2x: keep the target buffering in the middle of the max amount of queuable frames
let available_buffering_frames = (2. * config.max_buffering_frames).ceil() as usize;

let mime = match config.codec {
CodecType::H264 => "video/avc",
CodecType::Hevc => "video/hevc",
};

let format = MediaFormat::new();
format.set_str("mime", mime);
format.set_i32("width", 512);
format.set_i32("height", 1024);
format.set_buffer("csd-0", &csd_0);

for (key, value) in &config.options {
match value {
MediacodecDataType::Float(value) => format.set_f32(key, *value),
MediacodecDataType::Int32(value) => format.set_i32(key, *value),
MediacodecDataType::Int64(value) => format.set_i64(key, *value),
MediacodecDataType::String(value) => format.set_str(key, value),
}
}

let mut image_reader = ImageReader::new_with_usage(
1,
1,
Expand Down Expand Up @@ -242,56 +259,92 @@ pub fn video_decoder_split(
.set_buffer_removed_listener(Box::new(|_, _| ()))
.unwrap();

let decoder = Arc::new(FakeThreadSafe(
MediaCodec::from_decoder_type(&mime).unwrap(),
));
let mime = mime_for_codec(config.codec);

let format = MediaFormat::new();
format.set_str("mime", mime);
// Given https://github.com/alvr-org/ALVR/pull/1933#discussion_r1431902906 - change at own risk.
// It might be harmless, it might not be, but it's definitely a risk.
format.set_i32("width", 512);
format.set_i32("height", 1024);
format.set_buffer("csd-0", &csd_0);

for (key, value) in &config.options {
match value {
MediacodecDataType::Float(value) => format.set_f32(key, *value),
MediacodecDataType::Int32(value) => format.set_i32(key, *value),
MediacodecDataType::Int64(value) => format.set_i64(key, *value),
MediacodecDataType::String(value) => format.set_str(key, value),
}
}

info!("Using AMediaCoded format:{} ", format);
decoder
.configure(
&format,
Some(&image_reader.window().unwrap()),
MediaCodecDirection::Decoder,
)
.unwrap();
decoder.start().unwrap();

{
let mut decoder_lock = decoder_sink.lock();
let preparing_decoder = if config.force_software_decoder {
decoder_attempt_setup(config.codec, true, &format, &image_reader)
} else {
// Hardware decoders sometimes fail at the CSD-0.
// May as well fall back if this occurs.
match decoder_attempt_setup(config.codec, false, &format, &image_reader) {
Ok(d) => Ok(d),
Err(e) => {
// would be "warn!" but this is a severe caveat and a pretty major error.
error!(
"Attempting software fallback due to error in default decoder: {:#}",
e
);
decoder_attempt_setup(config.codec, true, &format, &image_reader)
}
}
};

match preparing_decoder {
Ok(prepared_decoder) => {
let decoder = Arc::new(FakeThreadSafe(prepared_decoder));

*decoder_lock = Some(Arc::clone(&decoder));
{
let mut decoder_lock = decoder_sink.lock();

decoder_ready_notifier.notify_one();
}
*decoder_lock = Some(Arc::clone(&decoder));

while running.value() {
match decoder.dequeue_output_buffer(Duration::from_millis(1)) {
Ok(DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
// The buffer timestamp is actually nanoseconds
let presentation_time_ns = buffer.info().presentation_time_us();
decoder_ready_notifier.notify_one();
}

while running.value() {
match decoder.dequeue_output_buffer(Duration::from_millis(1)) {
Ok(DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
// The buffer timestamp is actually nanoseconds
let presentation_time_ns = buffer.info().presentation_time_us();

if let Err(e) = decoder
.release_output_buffer_at_time(buffer, presentation_time_ns)
{
error!("Decoder dequeue error: {e}");
}
}
Ok(DequeuedOutputBufferInfoResult::TryAgainLater) => {
thread::yield_now()
}
Ok(i) => info!("Decoder dequeue event: {i:?}"),
Err(e) => {
error!("Decoder dequeue error: {e}");

if let Err(e) =
decoder.release_output_buffer_at_time(buffer, presentation_time_ns)
{
error!("Decoder dequeue error: {e}");
// lessen logcat flood (just in case)
thread::sleep(Duration::from_millis(50));
}
}
}
Ok(DequeuedOutputBufferInfoResult::TryAgainLater) => thread::yield_now(),
Ok(i) => info!("Decoder dequeue event: {i:?}"),
Err(e) => {
error!("Decoder dequeue error: {e}");

// lessen logcat flood (just in case)
thread::sleep(Duration::from_millis(50));
}
// Destroy all resources
decoder_sink.lock().take(); // Make sure the shared ref is deleted first
decoder.stop().unwrap();
drop(decoder);
}
Err(e) => {
show_e(e);
}
}

// Destroy all resources
decoder_sink.lock().take(); // Make sure the shared ref is deleted first
decoder.stop().unwrap();
drop(decoder);

image_queue.lock().clear();
error!("FIXME: Leaking Imagereader!");
Box::leak(Box::new(image_reader));
Expand Down
6 changes: 6 additions & 0 deletions alvr/session/src/settings.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,11 @@ pub struct VideoConfig {
#[schema(flag = "steamvr-restart")]
pub encoder_config: EncoderConfig,

#[schema(strings(
help = "Attempts to use a software decoder on the device. Slow, but may work around broken codecs."
))]
pub force_software_decoder: bool,

pub mediacodec_extra_options: Vec<(String, MediacodecDataType)>,

#[schema(flag = "steamvr-restart")]
Expand Down Expand Up @@ -1294,6 +1299,7 @@ pub fn session_settings_default() -> SettingsDefault {
vertical_offset_deg: 0.0,
},
},
force_software_decoder: false,
color_correction: SwitchDefault {
enabled: true,
content: ColorCorrectionConfigDefault {
Expand Down

0 comments on commit 377c556

Please sign in to comment.