Skip to content

Commit

Permalink
Show all samples/frames in a video in a nice table (#8102)
Browse files Browse the repository at this point in the history 
### What
Add a table showing all the samples (=frames) in a video:


![image](https://github.com/user-attachments/assets/2808ed30-5787-47a1-ab77-7d76f28ec0c7)


![image](https://github.com/user-attachments/assets/c879a4ec-cb03-4884-a979-5949363c85c8)


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/8102?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/8102?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/8102)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.
  • Loading branch information
@emilk
emilk authored Nov 12, 2024
1 parent 93b611e commit d2e0220
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 96 deletions.
4 changes: 2 additions & 2 deletions crates/store/re_video/src/demux/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ impl VideoData {
// Segments are guaranteed to be sorted among each other, but within a segment,
// presentation timestamps may not be sorted since this is sorted by decode timestamps.
self.gops.iter().flat_map(|seg| {
self.samples[seg.decode_time_range()]
self.samples[seg.sample_range_usize()]
.iter()
.map(|sample| sample.presentation_timestamp)
.sorted()
Expand Down Expand Up @@ -432,7 +432,7 @@ pub struct GroupOfPictures {

impl GroupOfPictures {
/// The GOP's `sample_range` mapped to `usize` for slicing.
pub fn decode_time_range(&self) -> Range<usize> {
pub fn sample_range_usize(&self) -> Range<usize> {
Range {
start: self.sample_range.start as usize,
end: self.sample_range.end as usize,
Expand Down
4 changes: 2 additions & 2 deletions crates/viewer/re_data_ui/src/blob.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use re_viewer_context::UiLayout;

use crate::{
image::image_preview_ui,
video::{show_decoded_frame_info, show_video_blob_info},
video::{show_decoded_frame_info, video_result_ui},
EntityDataUi,
};

Expand Down Expand Up @@ -131,7 +131,7 @@ pub fn blob_preview_and_save_ui(
ctx.app_options.video_decoder_hw_acceleration,
)
});
show_video_blob_info(ui, ui_layout, &video_result);
video_result_ui(ui, ui_layout, &video_result);
video_result_for_frame_preview = Some(video_result);
}
}
Expand Down
265 changes: 174 additions & 91 deletions crates/viewer/re_data_ui/src/video.rs
View file
Original file line number Diff line number Diff line change
@@ -1,109 +1,28 @@
use egui_extras::Column;
use re_renderer::{
external::re_video::VideoLoadError, resource_managers::SourceImageDataFormat,
video::VideoFrameTexture,
};
use re_types::components::VideoTimestamp;
use re_ui::{list_item::PropertyContent, UiExt};
use re_video::{decode::FrameInfo, demux::SamplesStatistics};
use re_ui::{list_item::PropertyContent, DesignTokens, UiExt};
use re_video::{decode::FrameInfo, demux::SamplesStatistics, VideoData};
use re_viewer_context::UiLayout;

pub fn show_video_blob_info(
pub fn video_result_ui(
ui: &mut egui::Ui,
ui_layout: UiLayout,
video_result: &Result<re_renderer::video::Video, VideoLoadError>,
) {
re_tracing::profile_function!();

#[allow(clippy::match_same_arms)]
match video_result {
Ok(video) => {
if ui_layout.is_single_line() {
return;
if !ui_layout.is_single_line() {
re_ui::list_item::list_item_scope(ui, "video_blob_info", |ui| {
video_data_ui(ui, ui_layout, video.data());
});
}

let data = video.data();

re_ui::list_item::list_item_scope(ui, "video_blob_info", |ui| {
ui.list_item_flat_noninteractive(
PropertyContent::new("Dimensions").value_text(format!(
"{}x{}",
data.width(),
data.height()
)),
);
if let Some(bit_depth) = data.config.stsd.contents.bit_depth() {
ui.list_item_flat_noninteractive(PropertyContent::new("Bit depth").value_fn(
|ui, _| {
ui.label(bit_depth.to_string());
if 8 < bit_depth {
// TODO(#7594): HDR videos
ui.warning_label("HDR").on_hover_ui(|ui| {
ui.label(
"High-dynamic-range videos not yet supported by Rerun",
);
ui.hyperlink("https://github.com/rerun-io/rerun/issues/7594");
});
}
if data.is_monochrome() == Some(true) {
ui.label("(monochrome)");
}
},
));
}
if let Some(subsampling_mode) = data.subsampling_mode() {
// Don't show subsampling mode for monochrome, doesn't make sense usually.
if data.is_monochrome() != Some(true) {
ui.list_item_flat_noninteractive(
PropertyContent::new("Subsampling")
.value_text(subsampling_mode.to_string()),
);
}
}
ui.list_item_flat_noninteractive(
PropertyContent::new("Duration")
.value_text(format!("{}", re_log_types::Duration::from(data.duration()))),
);
// Some people may think that num_frames / duration = fps, but that's not true, videos may have variable frame rate.
// Video containers and codecs like talking about samples or chunks rather than frames, but for how we define a chunk today,
// a frame is always a single chunk of data is always a single sample, see [`re_video::decode::Chunk`].
// So for all practical purposes the sample count _is_ the number of frames, at least how we use it today.
ui.list_item_flat_noninteractive(
PropertyContent::new("Frame count")
.value_text(re_format::format_uint(data.num_samples())),
);
ui.list_item_flat_noninteractive(
PropertyContent::new("Codec").value_text(data.human_readable_codec_string()),
);

if ui_layout != UiLayout::Tooltip {
ui.list_item_collapsible_noninteractive_label("MP4 tracks", true, |ui| {
for (track_id, track_kind) in &data.mp4_tracks {
let track_kind_string = match track_kind {
Some(re_video::TrackKind::Audio) => "audio",
Some(re_video::TrackKind::Subtitle) => "subtitle",
Some(re_video::TrackKind::Video) => "video",
None => "unknown",
};
ui.list_item_flat_noninteractive(
PropertyContent::new(format!("Track {track_id}"))
.value_text(track_kind_string),
);
}
});
ui.list_item_collapsible_noninteractive_label(
"More video statistics",
false,
|ui| {
ui.list_item_flat_noninteractive(
PropertyContent::new("Number of GOPs")
.value_text(data.gops.len().to_string()),
)
.on_hover_text(
"The total number of Group of Pictures (GOPs) in the video.",
);
samples_statistics_ui(ui, &data.samples_statistics);
},
);
}
});
}
Err(VideoLoadError::MimeTypeIsNotAVideo { .. }) => {
// Don't show an error if this wasn't a video in the first place.
Expand All @@ -126,6 +45,170 @@ pub fn show_video_blob_info(
}
}

fn video_data_ui(ui: &mut egui::Ui, ui_layout: UiLayout, video_data: &VideoData) {
re_tracing::profile_function!();

ui.list_item_flat_noninteractive(PropertyContent::new("Dimensions").value_text(format!(
"{}x{}",
video_data.width(),
video_data.height()
)));
if let Some(bit_depth) = video_data.config.stsd.contents.bit_depth() {
ui.list_item_flat_noninteractive(PropertyContent::new("Bit depth").value_fn(|ui, _| {
ui.label(bit_depth.to_string());
if 8 < bit_depth {
// TODO(#7594): HDR videos
ui.warning_label("HDR").on_hover_ui(|ui| {
ui.label("High-dynamic-range videos not yet supported by Rerun");
ui.hyperlink("https://github.com/rerun-io/rerun/issues/7594");
});
}
if video_data.is_monochrome() == Some(true) {
ui.label("(monochrome)");
}
}));
}
if let Some(subsampling_mode) = video_data.subsampling_mode() {
// Don't show subsampling mode for monochrome, doesn't make sense usually.
if video_data.is_monochrome() != Some(true) {
ui.list_item_flat_noninteractive(
PropertyContent::new("Subsampling").value_text(subsampling_mode.to_string()),
);
}
}
ui.list_item_flat_noninteractive(PropertyContent::new("Duration").value_text(format!(
"{}",
re_log_types::Duration::from(video_data.duration())
)));
// Some people may think that num_frames / duration = fps, but that's not true, videos may have variable frame rate.
// Video containers and codecs like talking about samples or chunks rather than frames, but for how we define a chunk today,
// a frame is always a single chunk of data is always a single sample, see [`re_video::decode::Chunk`].
// So for all practical purposes the sample count _is_ the number of frames, at least how we use it today.
ui.list_item_flat_noninteractive(
PropertyContent::new("Frame count")
.value_text(re_format::format_uint(video_data.num_samples())),
);
ui.list_item_flat_noninteractive(
PropertyContent::new("Codec").value_text(video_data.human_readable_codec_string()),
);

if ui_layout != UiLayout::Tooltip {
ui.list_item_collapsible_noninteractive_label("MP4 tracks", true, |ui| {
for (track_id, track_kind) in &video_data.mp4_tracks {
let track_kind_string = match track_kind {
Some(re_video::TrackKind::Audio) => "audio",
Some(re_video::TrackKind::Subtitle) => "subtitle",
Some(re_video::TrackKind::Video) => "video",
None => "unknown",
};
ui.list_item_flat_noninteractive(
PropertyContent::new(format!("Track {track_id}")).value_text(track_kind_string),
);
}
});

ui.list_item_collapsible_noninteractive_label("More video statistics", false, |ui| {
ui.list_item_flat_noninteractive(
PropertyContent::new("Number of GOPs")
.value_text(video_data.gops.len().to_string()),
)
.on_hover_text("The total number of Group of Pictures (GOPs) in the video.");
samples_statistics_ui(ui, &video_data.samples_statistics);
});

ui.list_item_collapsible_noninteractive_label("Video samples", false, |ui| {
samples_table_ui(ui, video_data);
});
}
}

fn samples_table_ui(ui: &mut egui::Ui, video_data: &VideoData) {
re_tracing::profile_function!();

egui_extras::TableBuilder::new(ui)
.auto_shrink([false, true])
.vscroll(true)
.max_scroll_height(800.0)
.columns(Column::auto(), 7)
.cell_layout(egui::Layout::left_to_right(egui::Align::Center))
.header(DesignTokens::table_header_height(), |mut header| {
DesignTokens::setup_table_header(&mut header);
header.col(|ui| {
ui.strong("Sample");
});
header.col(|ui| {
ui.strong("GOP");
});
header.col(|ui| {
ui.strong("Sync");
});
header.col(|ui| {
ui.strong("DTS").on_hover_text("Decode timestamp");
});
header.col(|ui| {
ui.strong("PTS").on_hover_text("Presentation timestamp");
});
header.col(|ui| {
ui.strong("Duration");
});
header.col(|ui| {
ui.strong("Size");
});
})
.body(|mut body| {
DesignTokens::setup_table_body(&mut body);

body.rows(
DesignTokens::table_line_height(),
video_data.samples.len(),
|mut row| {
let sample_idx = row.index();
let sample = &video_data.samples[sample_idx];
let re_video::Sample {
is_sync,
decode_timestamp,
presentation_timestamp,
duration,
byte_offset: _,
byte_length,
} = *sample;

row.col(|ui| {
ui.monospace(sample_idx.to_string());
});
row.col(|ui| {
if let Some(gop_index) = video_data
.gop_index_containing_presentation_timestamp(presentation_timestamp)
{
ui.monospace(re_format::format_uint(gop_index));
}
});
row.col(|ui| {
if is_sync {
ui.label("sync");
}
});
row.col(|ui| {
ui.monospace(re_format::format_int(decode_timestamp.0));
});
row.col(|ui| {
ui.monospace(re_format::format_int(presentation_timestamp.0));
});

row.col(|ui| {
ui.monospace(
re_log_types::Duration::from(duration.duration(video_data.timescale))
.to_string(),
);
});
row.col(|ui| {
ui.monospace(re_format::format_bytes(byte_length as _));
});
},
);
});
}

pub fn show_decoded_frame_info(
render_ctx: Option<&re_renderer::RenderContext>,
ui: &mut egui::Ui,
Expand Down
2 changes: 1 addition & 1 deletion crates/viewer/re_renderer/src/video/player.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ impl VideoPlayer {
return Ok(());
};

let samples = &self.data.samples[gop.decode_time_range()];
let samples = &self.data.samples[gop.sample_range_usize()];

for sample in samples {
let chunk = sample.get(video_data).ok_or(VideoPlayerError::BadData)?;
Expand Down

0 comments on commit d2e0220

Please sign in to comment.