Merge branch 'main' into benliepert/update_ndarray

Author: benliepert

.typos.toml

@@ -34,6 +34,7 @@ armour = "armor"
 artefact = "artifact"
 authorise = "authorize"
 behaviour = "behavior"
+behavioural = "behavioral"
 British = "American"
 calibre = "caliber"
 # allow 'cancelled' since Arrow uses it.

BUILD.md

@@ -50,6 +50,9 @@ All Rust examples are set up as separate executables, so they can be run by spec
 cargo run -p dna
 ```
 
+They will either connect to an already running rerun viewer, or spawn a new one.
+In debug builds, it will spawn `target/debug/rerun` if it exists, otherwise look for `rerun` on `PATH`.
+
 
 ## Building and installing the Rerun Python SDK
 

Cargo.lock

@@ -1777,7 +1777,7 @@ checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30"
 [[package]]
 name = "ecolor"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "bytemuck",
  "emath",
@@ -1787,7 +1787,7 @@ dependencies = [
 [[package]]
 name = "eframe"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "ahash",
  "bytemuck",
@@ -1824,7 +1824,7 @@ dependencies = [
 [[package]]
 name = "egui"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "accesskit",
  "ahash",
@@ -1841,7 +1841,7 @@ dependencies = [
 [[package]]
 name = "egui-wgpu"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "ahash",
  "bytemuck",
@@ -1860,7 +1860,7 @@ dependencies = [
 [[package]]
 name = "egui-winit"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "accesskit_winit",
  "ahash",
@@ -1879,30 +1879,28 @@ dependencies = [
 [[package]]
 name = "egui_commonmark"
 version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe88871b75bd43c52a2b44ce5b53160506e7976e239112c56728496d019cc60d"
+source = "git+https://github.com/rerun-io/egui_commonmark?rev=7a9dc755bfa351a3796274cb8ca87129b051c084#7a9dc755bfa351a3796274cb8ca87129b051c084"
 dependencies = [
  "egui",
  "egui_commonmark_backend",
  "egui_extras",
- "pulldown-cmark 0.11.0",
+ "pulldown-cmark 0.12.1",
 ]
 
 [[package]]
 name = "egui_commonmark_backend"
 version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "148edd9546feba319b16d5a5e551cda46095031ec1e6665e5871eef9ee692967"
+source = "git+https://github.com/rerun-io/egui_commonmark?rev=7a9dc755bfa351a3796274cb8ca87129b051c084#7a9dc755bfa351a3796274cb8ca87129b051c084"
 dependencies = [
  "egui",
  "egui_extras",
- "pulldown-cmark 0.11.0",
+ "pulldown-cmark 0.12.1",
 ]
 
 [[package]]
 name = "egui_extras"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "ahash",
  "egui",
@@ -1918,13 +1916,13 @@ dependencies = [
 [[package]]
 name = "egui_glow"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "ahash",
  "bytemuck",
  "egui",
  "egui-winit",
- "glow",
+ "glow 0.14.0",
  "log",
  "memoffset 0.9.0",
  "puffin",
@@ -1947,7 +1945,7 @@ dependencies = [
 [[package]]
 name = "egui_table"
 version = "0.28.1"
-source = "git+https://github.com/rerun-io/egui_table.git?rev=0f594701d528c4a9553521cb941de1886549dc70#0f594701d528c4a9553521cb941de1886549dc70"
+source = "git+https://github.com/rerun-io/egui_table.git?rev=c76473b244f03a7c67fbbbff9def6fc86c1ca4ea#c76473b244f03a7c67fbbbff9def6fc86c1ca4ea"
 dependencies = [
  "egui",
  "serde",
@@ -1991,7 +1989,7 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 [[package]]
 name = "emath"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "bytemuck",
  "serde",
@@ -2092,7 +2090,7 @@ dependencies = [
 [[package]]
 name = "epaint"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 dependencies = [
  "ab_glyph",
  "ahash",
@@ -2111,7 +2109,7 @@ dependencies = [
 [[package]]
 name = "epaint_default_fonts"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
+source = "git+https://github.com/emilk/egui.git?rev=e0f0b7f47f4094967530214aec060b9f1ce5954e#e0f0b7f47f4094967530214aec060b9f1ce5954e"
 
 [[package]]
 name = "equivalent"
@@ -2550,6 +2548,18 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "glow"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f865cbd94bd355b89611211e49508da98a1fce0ad755c1e8448fb96711b24528"
+dependencies = [
+ "js-sys",
+ "slotmap",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "gltf"
 version = "1.4.1"
@@ -3474,7 +3484,7 @@ dependencies = [
 [[package]]
 name = "mp4"
 version = "0.14.0"
-source = "git+https://github.com/rerun-io/mp4?rev=ef529032547d7f97161e95c58bd76856cb116349#ef529032547d7f97161e95c58bd76856cb116349"
+source = "git+https://github.com/rerun-io/mp4?rev=b019fcf3ac8e1befded54728686997cc168117c8#b019fcf3ac8e1befded54728686997cc168117c8"
 dependencies = [
  "byteorder",
  "bytes",
@@ -4526,9 +4536,9 @@ dependencies = [
 
 [[package]]
 name = "pulldown-cmark"
-version = "0.11.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8746739f11d39ce5ad5c2520a9b75285310dbfe78c541ccf832d38615765aec0"
+checksum = "666f0f59e259aea2d72e6012290c09877a780935cc3c18b1ceded41f3890d59c"
 dependencies = [
  "bitflags 2.6.0",
  "memchr",
@@ -4947,7 +4957,6 @@ dependencies = [
  "re_smart_channel",
  "re_tracing",
  "re_types",
- "re_video",
  "thiserror",
  "walkdir",
 ]
@@ -5480,6 +5489,7 @@ dependencies = [
  "re_data_ui",
  "re_dataframe",
  "re_entity_db",
+ "re_format",
  "re_log",
  "re_log_types",
  "re_renderer",
@@ -5505,6 +5515,7 @@ dependencies = [
  "egui",
  "glam",
  "hexasphere",
+ "image",
  "itertools 0.13.0",
  "mimalloc",
  "nohash-hasher",
@@ -5799,8 +5810,10 @@ dependencies = [
 name = "re_video"
 version = "0.19.0-alpha.1+dev"
 dependencies = [
+ "itertools 0.13.0",
  "mp4",
  "ordered-float",
+ "thiserror",
 ]
 
 [[package]]
@@ -6156,6 +6169,7 @@ dependencies = [
  "re_arrow2",
  "re_log",
  "re_sdk",
+ "re_video",
 ]
 
 [[package]]
@@ -6180,6 +6194,7 @@ dependencies = [
  "re_log_types",
  "re_memory",
  "re_sdk",
+ "re_video",
  "re_web_viewer_server",
  "re_ws_comms",
  "uuid",
@@ -7108,6 +7123,16 @@ dependencies = [
  "rerun",
 ]
 
+[[package]]
+name = "test_ui_wakeup"
+version = "0.19.0-alpha.1+dev"
+dependencies = [
+ "anyhow",
+ "clap",
+ "re_log",
+ "rerun",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.63"
@@ -7992,7 +8017,7 @@ dependencies = [
  "block",
  "cfg_aliases 0.1.1",
  "core-graphics-types",
- "glow",
+ "glow 0.13.1",
  "glutin_wgl_sys",
  "gpu-alloc",
  "gpu-allocator",

Cargo.toml

@@ -514,12 +514,12 @@ missing_errors_doc = "allow"
 # As a last resport, patch with a commit to our own repository.
 # ALWAYS document what PR the commit hash is part of, or when it was merged into the upstream trunk.
 
-ecolor = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }      # egui master 2024-09-06
-eframe = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }      # egui master 2024-09-06
-egui = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }        # egui master 2024-09-06
-egui_extras = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06
-egui-wgpu = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }   # egui master 2024-09-06
-emath = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }       # egui master 2024-09-06
+ecolor = { git = "https://github.com/emilk/egui.git", rev = "e0f0b7f47f4094967530214aec060b9f1ce5954e" }      # egui master 2024-09-19
+eframe = { git = "https://github.com/emilk/egui.git", rev = "e0f0b7f47f4094967530214aec060b9f1ce5954e" }      # egui master 2024-09-19
+egui = { git = "https://github.com/emilk/egui.git", rev = "e0f0b7f47f4094967530214aec060b9f1ce5954e" }        # egui master 2024-09-19
+egui_extras = { git = "https://github.com/emilk/egui.git", rev = "e0f0b7f47f4094967530214aec060b9f1ce5954e" } # egui master 2024-09-19
+egui-wgpu = { git = "https://github.com/emilk/egui.git", rev = "e0f0b7f47f4094967530214aec060b9f1ce5954e" }   # egui master 2024-09-19
+emath = { git = "https://github.com/emilk/egui.git", rev = "e0f0b7f47f4094967530214aec060b9f1ce5954e" }       # egui master 2024-09-19
 
 # Useful while developing:
 # ecolor = { path = "../../egui/crates/ecolor" }
@@ -535,11 +535,11 @@ emath = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf
 egui_tiles = { git = "https://github.com/rerun-io/egui_tiles", rev = "b2f5e232524deed983bcad01c05f27d0e8b89d98" } # https://github.com/rerun-io/egui_tiles/pull/78 2024-08-28
 # egui_tiles = { path = "../egui_tiles" }
 
-# egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark", rev = "63d5c8933445b9ea9088c4a50b71f4ede1f3c247" } # https://github.com/lampsitter/egui_commonmark/pull/51
+egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark", rev = "7a9dc755bfa351a3796274cb8ca87129b051c084" } # https://github.com/lampsitter/egui_commonmark/pull/65
 
 # commit on `rerun-io/mp4` `main` branch
-# https://github.com/rerun-io/mp4/commit/ef529032547d7f97161e95c58bd76856cb116349
-mp4 = { git = "https://github.com/rerun-io/mp4", rev = "ef529032547d7f97161e95c58bd76856cb116349" }
+# https://github.com/rerun-io/mp4/commit/b019fcf3ac8e1befded54728686997cc168117c8
+mp4 = { git = "https://github.com/rerun-io/mp4", rev = "b019fcf3ac8e1befded54728686997cc168117c8" }
 
 # commit on `rerun-io/re_arrow2` `main` branch
 # https://github.com/rerun-io/re_arrow2/commit/e4717d6debc6d4474ec10db8f629f823f57bad07

crates/build/re_dev_tools/src/build_examples/snippets.rs

@@ -36,8 +36,11 @@ impl Snippets {
         let snippet_root = snippets_dir.join("all");
         let snippets = collect_snippets_recursively(&snippet_root, &config, &snippet_root)?;
 
-        println!("Running {} snippets…", snippets.len());
+        println!("Download test assets…");
         let progress = MultiProgress::new();
+        download_test_assets(&progress)?;
+
+        println!("Running {} snippets…", snippets.len());
         let results: Vec<anyhow::Result<PathBuf>> = snippets
             .into_par_iter()
             .map(|example| example.build(&progress, &self.output_dir))
@@ -190,3 +193,12 @@ struct OptOut {
     /// example name -> languages
     run: HashMap<String, Vec<String>>,
 }
+
+fn download_test_assets(progress: &MultiProgress) -> anyhow::Result<()> {
+    let download_script = re_build_tools::cargo_metadata()?
+        .workspace_root
+        .join("tests/assets/download_test_assets.py");
+    let mut cmd = Command::new("python3");
+    cmd.arg(download_script.as_str());
+    wait_for_output(cmd, "download test assets", progress)
+}

crates/build/re_types_builder/src/codegen/cpp/mod.rs

@@ -1505,7 +1505,13 @@ fn to_arrow_method(
         (
             true,
             quote! {
-                return Loggable<#forwarded_type>::to_arrow(&instances->#field_name, num_instances);
+                if (num_instances == 0) {
+                    return Loggable<#forwarded_type>::to_arrow(nullptr, 0);
+                } else if (instances == nullptr) {
+                    return rerun::Error(ErrorCode::UnexpectedNullArgument, "Passed array instances is null when num_elements > 0.");
+                } else {
+                    return Loggable<#forwarded_type>::to_arrow(&instances->#field_name, num_instances);
+                }
             },
         )
     } else {

crates/store/README.md

@@ -1 +1 @@
-Creates related to storing, indexing, trasmitting, and handling data.
+Creates related to storing, indexing, transmitting, and handling data.

crates/store/re_chunk/src/util.rs

@@ -82,7 +82,7 @@ pub fn arrays_to_list_array(
 // TODO(cmc): A possible improvement would be to pick the smallest key datatype possible based
 // on the cardinality of the input arrays.
 pub fn arrays_to_dictionary<Idx: Copy + Eq>(
-    array_datatype: ArrowDatatype,
+    array_datatype: &ArrowDatatype,
     arrays: &[Option<(Idx, &dyn ArrowArray)>],
 ) -> Option<ArrowDictionaryArray<i32>> {
     // Dedupe the input arrays based on the given primary key.
@@ -115,33 +115,29 @@ pub fn arrays_to_dictionary<Idx: Copy + Eq>(
     };
 
     // Concatenate the underlying data as usual, except only the _unique_ values!
+    // We still need the underlying data to be a list-array, so the dictionary's keys can index
+    // into this list-array.
     let data = if arrays_dense_deduped.is_empty() {
         arrow2::array::new_empty_array(array_datatype.clone())
     } else {
-        arrow2::compute::concatenate::concatenate(&arrays_dense_deduped)
+        let values = arrow2::compute::concatenate::concatenate(&arrays_dense_deduped)
             .map_err(|err| {
                 re_log::warn_once!("failed to concatenate arrays: {err}");
                 err
             })
-            .ok()?
-    };
-
-    // We still need the underlying data to be a list-array, so the dictionary's keys can index
-    // into this list-array.
-    let data = {
-        let datatype = ArrowListArray::<i32>::default_datatype(array_datatype);
+            .ok()?;
 
         #[allow(clippy::unwrap_used)] // yes, these are indeed lengths
         let offsets =
             ArrowOffsets::try_from_lengths(arrays_dense_deduped.iter().map(|array| array.len()))
                 .unwrap();
 
-        ArrowListArray::<i32>::new(datatype, offsets.into(), data, None)
+        ArrowListArray::<i32>::new(array_datatype.clone(), offsets.into(), values, None).to_boxed()
     };
 
     let datatype = ArrowDatatype::Dictionary(
         arrow2::datatypes::IntegerType::Int32,
-        std::sync::Arc::new(data.data_type().clone()),
+        std::sync::Arc::new(array_datatype.clone()),
         true, // is_sorted
     );