Web-only video support

.vscode/settings.json

@@ -42,9 +42,16 @@
     },
     "rust-analyzer.diagnostics.enable": false, // False positives
     "rust-analyzer.showUnlinkedFileNotification": false,
-    // Uncomment the following option and restart rust-analyzer to get it to check code behind `cfg(target_arch=wasm32)`.
+
+    // Uncomment the following options and restart rust-analyzer to get it to check code behind `cfg(target_arch=wasm32)`.
     // Don't forget to put it in a comment again before committing.
-    // "rust-analyzer.cargo.target": "wasm32-unknown-unknown"
+    "rust-analyzer.cargo.target": "wasm32-unknown-unknown",
+    "rust-analyzer.cargo.cfgs": {
+        "web": null,
+        "webgl": null,
+        "webgpu": null,
+    },
+
     "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", // Use cmake-tools to grab configs.
     "C_Cpp.autoAddFileAssociations": false,
     "cmake.buildDirectory": "${workspaceRoot}/build/debug",

ARCHITECTURE.md

@@ -216,6 +216,7 @@ Update instructions:
 | re_string_interner | Yet another string interning library                                                 |
 | re_tracing         | Helpers for tracing/spans/flamegraphs and such.                                      |
 | re_tuid            | 128-bit Time-based Unique Identifier                                                 |
+| re_video           | Video decoding library                                                               |
 
 
 

Cargo.lock

@@ -3146,6 +3146,19 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "mp4"
+version = "0.14.0"
+source = "git+https://github.com/rerun-io/mp4?rev=523ebcc71032e7022335f61d7314b89f6068208a#523ebcc71032e7022335f61d7314b89f6068208a"
+dependencies = [
+ "byteorder",
+ "bytes",
+ "num-rational",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "multimap"
 version = "0.10.0"
@@ -3286,6 +3299,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-complex"
 version = "0.4.3"
@@ -3316,11 +3340,23 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-rational"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+ "serde",
+]
+
 [[package]]
 name = "num-traits"
-version = "0.2.15"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 dependencies = [
  "autocfg",
  "libm",
@@ -4746,6 +4782,7 @@ dependencies = [
  "gltf",
  "half 2.3.1",
  "itertools 0.13.0",
+ "js-sys",
  "never",
  "notify",
  "ordered-float",
@@ -4758,6 +4795,7 @@ dependencies = [
  "re_log",
  "re_math",
  "re_tracing",
+ "re_video",
  "serde",
  "slotmap",
  "smallvec",
@@ -4768,9 +4806,12 @@ dependencies = [
  "type-map",
  "unindent",
  "walkdir",
+ "wasm-bindgen",
  "wasm-bindgen-futures",
+ "web-sys",
  "wgpu",
  "wgpu-core",
+ "wgpu-types",
 ]
 
 [[package]]
@@ -5157,6 +5198,7 @@ dependencies = [
  "re_tracing",
  "re_types_builder",
  "re_types_core",
+ "re_video",
  "serde",
  "similar-asserts",
  "smallvec",
@@ -5249,6 +5291,14 @@ dependencies = [
  "sublime_fuzzy",
 ]
 
+[[package]]
+name = "re_video"
+version = "0.19.0-alpha.1+dev"
+dependencies = [
+ "mp4",
+ "ordered-float",
+]
+
 [[package]]
 name = "re_viewer"
 version = "0.19.0-alpha.1+dev"

Cargo.toml

@@ -53,6 +53,7 @@ re_sdk_comms = { path = "crates/store/re_sdk_comms", version = "=0.19.0-alpha.1"
 re_types = { path = "crates/store/re_types", version = "=0.19.0-alpha.1", default-features = false }
 re_types_blueprint = { path = "crates/store/re_types_blueprint", version = "=0.19.0-alpha.1", default-features = false }
 re_types_core = { path = "crates/store/re_types_core", version = "=0.19.0-alpha.1", default-features = false }
+re_video = { path = "crates/store/re_video", version = "=0.19.0-alpha.1", default-features = false }
 re_ws_comms = { path = "crates/store/re_ws_comms", version = "=0.19.0-alpha.1", default-features = false }
 
 # crates/top:
@@ -190,6 +191,7 @@ memory-stats = "1.1"
 mimalloc = "=0.1.37" # TODO(#5875): `mimalloc` starts leaking OS pages starting with `0.1.38`.
 mime_guess2 = "2.0" # infer MIME type by file extension, and map mime to file extension
 mint = "0.5.9"
+mp4 = "0.14.0"
 natord = "1.0.9"
 ndarray = "0.15"
 ndarray-rand = "0.14"
@@ -286,6 +288,7 @@ wgpu = { version = "0.20.1", default-features = false, features = [
   "fragile-send-sync-non-atomic-wasm",
 ] }
 wgpu-core = "0.21.0"
+wgpu-types = "0.20.0"
 xshell = "0.2"
 zip = { version = "0.6", default-features = false }
 
@@ -524,3 +527,7 @@ missing_errors_doc = "allow"
 # egui_tiles = { path = "../egui_tiles" }
 
 # egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark", rev = "63d5c8933445b9ea9088c4a50b71f4ede1f3c247" } # https://github.com/lampsitter/egui_commonmark/pull/51
+
+# commit on `rerun-io/mp4` `main` branch
+# https://github.com/rerun-io/mp4/commit/523ebcc71032e7022335f61d7314b89f6068208a
+mp4 = { git = "https://github.com/rerun-io/mp4", rev = "523ebcc71032e7022335f61d7314b89f6068208a" }

crates/build/re_types_builder/src/codegen/cpp/mod.rs

@@ -2296,6 +2296,13 @@ fn quote_obj_docs(objects: &Objects, obj: &Object) -> TokenStream {
         *first_line = format!("**{}**: {}", obj.kind.singular_name(), first_line);
     }
 
+    if obj.is_experimental() {
+        lines.push(String::new());
+        lines.push(
+            "⚠ **This type is experimental and may be removed in future versions**".to_owned(),
+        );
+    }
+
     quote_doc_lines(&lines)
 }
 

crates/build/re_types_builder/src/codegen/docs/mod.rs

@@ -230,6 +230,7 @@ fn object_page(
     views_per_archetype: &ViewsPerArchetype,
 ) -> String {
     let is_unreleased = object.is_attr_set(crate::ATTR_DOCS_UNRELEASED);
+    let is_experimental = object.is_experimental();
 
     let top_level_docs = object.docs.lines_for(objects, Target::WebDocsMarkdown);
 
@@ -254,6 +255,15 @@ fn object_page(
     write_frontmatter(&mut page, &title, None);
     putln!(page);
 
+    if is_experimental {
+        putln!(page);
+        putln!(
+            page,
+            "⚠️ **This type is experimental and may be removed in future versions**"
+        );
+        putln!(page);
+    }
+
     if let Some(deprecation_notice) = object.deprecation_notice() {
         putln!(
             page,

crates/build/re_types_builder/src/codegen/python/mod.rs

@@ -750,7 +750,7 @@ fn code_for_struct(
 
             // Generating docs for all the fields creates A LOT of visual noise in the API docs.
             let show_fields_in_docs = false;
-            let doc_lines = lines_from_docs(reporter, objects, &field.docs);
+            let doc_lines = lines_from_docs(reporter, objects, &field.docs, false);
             if !doc_lines.is_empty() {
                 if show_fields_in_docs {
                     code.push_indented(1, quote_doc_lines(doc_lines), 0);
@@ -866,7 +866,7 @@ fn code_for_enum(
 
         // Generating docs for all the fields creates A LOT of visual noise in the API docs.
         let show_fields_in_docs = true;
-        let doc_lines = lines_from_docs(reporter, objects, &variant.docs);
+        let doc_lines = lines_from_docs(reporter, objects, &variant.docs, false);
         if !doc_lines.is_empty() {
             if show_fields_in_docs {
                 code.push_indented(1, quote_doc_lines(doc_lines), 0);
@@ -1206,7 +1206,7 @@ fn quote_examples(examples: Vec<Example<'_>>, lines: &mut Vec<String>) {
 
 /// Ends with double newlines, unless empty.
 fn quote_obj_docs(reporter: &Reporter, objects: &Objects, obj: &Object) -> String {
-    let mut lines = lines_from_docs(reporter, objects, &obj.docs);
+    let mut lines = lines_from_docs(reporter, objects, &obj.docs, obj.is_experimental());
 
     if let Some(first_line) = lines.first_mut() {
         // Prefix with object kind:
@@ -1216,9 +1216,21 @@ fn quote_obj_docs(reporter: &Reporter, objects: &Objects, obj: &Object) -> Strin
     quote_doc_lines(lines)
 }
 
-fn lines_from_docs(reporter: &Reporter, objects: &Objects, docs: &Docs) -> Vec<String> {
+fn lines_from_docs(
+    reporter: &Reporter,
+    objects: &Objects,
+    docs: &Docs,
+    is_experimental: bool,
+) -> Vec<String> {
     let mut lines = docs.lines_for(objects, Target::Python);
 
+    if is_experimental {
+        lines.push(String::new());
+        lines.push(
+            "⚠️ **This type is experimental and may be removed in future versions**".to_owned(),
+        );
+    }
+
     let examples = collect_snippets_for_api_docs(docs, "py", true).unwrap_or_else(|err| {
         reporter.error_any(err);
         vec![]

crates/build/re_types_builder/src/codegen/rust/api.rs

@@ -563,6 +563,7 @@ fn quote_enum(
             &field.fqname,
             &field.docs,
             Target::Rust,
+            false,
         )
         .join("\n");
         if docstring_md.is_empty() {
@@ -660,6 +661,7 @@ fn quote_field_docs(reporter: &Reporter, objects: &Objects, field: &ObjectField)
         &field.fqname,
         &field.docs,
         Target::Rust,
+        false,
     );
 
     let require_field_docs = false;
@@ -678,6 +680,7 @@ fn quote_obj_docs(reporter: &Reporter, objects: &Objects, obj: &Object) -> Token
         &obj.fqname,
         &obj.docs,
         Target::Rust,
+        obj.is_experimental(),
     );
 
     // Prefix first line with `**Datatype**: ` etc:

crates/build/re_types_builder/src/codegen/rust/reflection.rs

@@ -108,6 +108,7 @@ fn generate_component_reflection(
             &obj.fqname,
             &obj.docs,
             Target::WebDocsMarkdown,
+            obj.is_experimental(),
         )
         .join("\n");
         let quoted_reflection = quote! {
@@ -155,6 +156,7 @@ fn generate_archetype_reflection(reporter: &Reporter, objects: &Objects) -> Toke
                 &field.fqname,
                 &field.docs,
                 Target::WebDocsMarkdown,
+                obj.is_experimental(),
             )
             .join("\n");
 
@@ -177,6 +179,7 @@ fn generate_archetype_reflection(reporter: &Reporter, objects: &Objects) -> Toke
             &obj.fqname,
             &obj.docs,
             Target::WebDocsMarkdown,
+            obj.is_experimental(),
         )
         .join("\n");
         let quoted_archetype_reflection = quote! {

crates/build/re_types_builder/src/codegen/rust/util.rs

@@ -317,9 +317,17 @@ pub fn doc_as_lines(
     fqname: &str,
     docs: &Docs,
     target: Target,
+    is_experimental: bool,
 ) -> Vec<String> {
     let mut lines = docs.lines_for(objects, target);
 
+    if is_experimental {
+        lines.push(String::new());
+        lines.push(
+            "⚠️ **This type is experimental and may be removed in future versions**".to_owned(),
+        );
+    }
+
     let examples = if !fqname.starts_with("rerun.blueprint.views") {
         collect_snippets_for_api_docs(docs, "rs", true)
             .map_err(|err| reporter.error(virtpath, fqname, err))

crates/build/re_types_builder/src/lib.rs

@@ -187,6 +187,7 @@ pub const ATTR_RERUN_OVERRIDE_TYPE: &str = "attr.rerun.override_type";
 pub const ATTR_RERUN_SCOPE: &str = "attr.rerun.scope";
 pub const ATTR_RERUN_VIEW_IDENTIFIER: &str = "attr.rerun.view_identifier";
 pub const ATTR_RERUN_DEPRECATED: &str = "attr.rerun.deprecated";
+pub const ATTR_RERUN_EXPERIMENTAL: &str = "attr.rerun.experimental";
 
 pub const ATTR_PYTHON_ALIASES: &str = "attr.python.aliases";
 pub const ATTR_PYTHON_ARRAY_ALIASES: &str = "attr.python.array_aliases";

crates/build/re_types_builder/src/objects.rs

@@ -587,6 +587,10 @@ impl Object {
         self.try_get_attr::<String>(crate::ATTR_RERUN_DEPRECATED)
     }
 
+    pub fn is_experimental(&self) -> bool {
+        self.is_attr_set(crate::ATTR_RERUN_EXPERIMENTAL)
+    }
+
     pub fn doc_category(&self) -> Option<String> {
         self.try_get_attr::<String>(crate::ATTR_DOCS_CATEGORY)
     }

crates/store/re_types/Cargo.toml

@@ -39,6 +39,9 @@ glam = ["dep:glam"]
 ## Integration with the [`image`](https://crates.io/crates/image/) crate, plus JPEG support.
 image = ["dep:ecolor", "dep:image"]
 
+## Conversion to/from our video format
+video = ["dep:re_video"]
+
 ## Enable (de)serialization using serde.
 serde = ["dep:serde"]
 
@@ -54,6 +57,7 @@ re_log.workspace = true
 re_log_types.workspace = true
 re_tracing.workspace = true
 re_types_core.workspace = true
+re_video = { workspace = true, optional = true }
 
 # External
 anyhow.workspace = true # TODO(#1845): Use thiserror instead

crates/store/re_types/definitions/rerun/archetypes/video.fbs

@@ -0,0 +1,24 @@
+namespace rerun.archetypes;
+
+/// A video file.
+///
+/// NOTE: Videos may only be visualized in the web viewer for now
+table AssetVideo (
+  "attr.rerun.experimental"
+) {
+  // --- Required ---
+
+  /// The asset's bytes.
+  blob: rerun.components.Blob ("attr.rerun.component_required", required, order: 1000);
+
+  // --- Recommended ---
+
+  /// The Media Type of the asset.
+  ///
+  /// Supported values:
+  /// * `video/mp4`
+  ///
+  /// If omitted, the viewer will try to guess from the data blob.
+  /// If it cannot guess, it won't be able to render the asset.
+  media_type: rerun.components.MediaType ("attr.rerun.component_recommended", nullable, order: 2000);
+}