Merge branch 'main' into emilk/native-av1-decode

.github/workflows/nightly.yml

@@ -4,8 +4,8 @@ on:
   workflow_dispatch:
   schedule:
     # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
-    # 12:15 UTC, every day
-    - cron: "15 12 * * *"
+    # 3am UTC, so 4am-5am CET and evening in east time, basically after everyone's day.
+    - cron: "15 3 * * *"
 
 defaults:
   run:
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: 1.76.0
+          toolchain: 1.79.0
 
       - run: cargo build -p rerun
 

.github/workflows/reusable_bench.yml

@@ -155,10 +155,13 @@ jobs:
           save-data-file: false
           auto-push: false
 
+      - uses: prefix-dev/setup-pixi@v0.8.1
+        with:
+          pixi-version: v0.25.0
+
       - name: Render benchmark result
         if: github.ref == 'refs/heads/main'
         run: |
-          pip install --break-system-packages google-cloud-storage==2.9.0
-          scripts/ci/render_bench.py crates \
+          pixi run python scripts/ci/render_bench.py crates \
             --after $(date -d"30 days ago" +%Y-%m-%d) \
             --output "gs://rerun-builds/graphs"

.github/workflows/reusable_track_size.yml

@@ -189,10 +189,13 @@ jobs:
           save-data-file: false
           auto-push: false
 
+      - uses: prefix-dev/setup-pixi@v0.8.1
+        with:
+          pixi-version: v0.25.0
+
       - name: Render benchmark result
         if: github.ref == 'refs/heads/main'
         run: |
-          python3 -m pip install --break-system-packages google-cloud-storage==2.9.0
-          scripts/ci/render_bench.py sizes \
+          pixi run python scripts/ci/render_bench.py sizes \
             --after $(date -d"180 days ago" +%Y-%m-%d) \
             --output "gs://rerun-builds/graphs"

ARCHITECTURE.md

@@ -164,6 +164,7 @@ Update instructions:
 | re_entity_db         | In-memory storage of Rerun entities                                      |
 | re_query             | Querying data in the re_chunk_store                                      |
 | re_dataframe         | The Rerun public data APIs.                                              |
+| re_dataframe2        | The Rerun public data APIs.                                              |
 | re_types             | The built-in Rerun data types, component types, and archetypes.          |
 | re_types_blueprint   | The core traits and types that power Rerun's Blueprint sub-system.       |
 | re_log_encoding      | Helpers for encoding and transporting Rerun log messages                 |

BUILD.md

@@ -23,12 +23,12 @@ cd rerun
 
 Now install the `pixi` package manager: <https://github.com/prefix-dev/pixi?tab=readme-ov-file#installation>
 
-Make sure `cargo --version` prints `1.76.0` once you are done.
+Make sure `cargo --version` prints `1.79.0` once you are done.
 
 If you are using an Apple-silicon Mac (M1, M2), make sure `rustc -vV` outputs `host: aarch64-apple-darwin`. If not, this should fix it:
 
 ```sh
-rustup set default-host aarch64-apple-darwin && rustup install 1.76.0
+rustup set default-host aarch64-apple-darwin && rustup install 1.79.0
 ```
 
 ## Building and running the Viewer

Cargo.lock

@@ -999,9 +999,9 @@ checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205"
 
 [[package]]
 name = "bytemuck"
-version = "1.13.1"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"
+checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
 dependencies = [
  "bytemuck_derive",
 ]
@@ -5132,6 +5132,33 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "re_dataframe2"
+version = "0.19.0-alpha.1+dev"
+dependencies = [
+ "ahash",
+ "anyhow",
+ "backtrace",
+ "indent",
+ "itertools 0.13.0",
+ "nohash-hasher",
+ "parking_lot",
+ "paste",
+ "re_arrow2",
+ "re_chunk",
+ "re_chunk_store",
+ "re_error",
+ "re_format",
+ "re_log",
+ "re_log_types",
+ "re_query",
+ "re_tracing",
+ "re_types",
+ "re_types_core",
+ "seq-macro",
+ "thiserror",
+]
+
 [[package]]
 name = "re_dev_tools"
 version = "0.19.0-alpha.1+dev"
@@ -5667,6 +5694,7 @@ dependencies = [
  "re_chunk_store",
  "re_data_ui",
  "re_log_types",
+ "re_query",
  "re_renderer",
  "re_space_view",
  "re_tracing",

Cargo.toml

@@ -24,7 +24,7 @@ homepage = "https://rerun.io"
 include = ["../../LICENSE-APACHE", "../../LICENSE-MIT", "**/*.rs", "Cargo.toml"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/rerun-io/rerun"
-rust-version = "1.76"
+rust-version = "1.79"
 version = "0.19.0-alpha.1+dev"
 
 [workspace.dependencies]
@@ -45,6 +45,7 @@ re_chunk_store = { path = "crates/store/re_chunk_store", version = "=0.19.0-alph
 re_data_loader = { path = "crates/store/re_data_loader", version = "=0.19.0-alpha.1", default-features = false }
 re_data_source = { path = "crates/store/re_data_source", version = "=0.19.0-alpha.1", default-features = false }
 re_dataframe = { path = "crates/store/re_dataframe", version = "=0.19.0-alpha.1", default-features = false }
+re_dataframe2 = { path = "crates/store/re_dataframe2", version = "=0.19.0-alpha.1", default-features = false }
 re_entity_db = { path = "crates/store/re_entity_db", version = "=0.19.0-alpha.1", default-features = false }
 re_format_arrow = { path = "crates/store/re_format_arrow", version = "=0.19.0-alpha.1", default-features = false }
 re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.19.0-alpha.1", default-features = false }
@@ -147,7 +148,7 @@ bincode = "1.3"
 bit-vec = "0.7"
 bitflags = { version = "2.4", features = ["bytemuck"] }
 blackbox = "0.2.0"
-bytemuck = { version = "1.11", features = ["extern_crate_alloc"] }
+bytemuck = { version = "1.18", features = ["extern_crate_alloc"] }
 camino = "1.1"
 cargo_metadata = "0.18"
 cargo-run-wasm = "0.3.2"
@@ -388,6 +389,7 @@ disallowed_types = "warn"                   # See clippy.toml
 doc_link_with_quotes = "warn"
 doc_markdown = "warn"
 empty_enum = "warn"
+empty_enum_variants_with_brackets = "warn"
 enum_glob_use = "warn"
 equatable_if_let = "warn"
 exit = "warn"
@@ -411,6 +413,8 @@ inefficient_to_string = "warn"
 infinite_loop = "warn"
 into_iter_without_iter = "warn"
 invalid_upcast_comparisons = "warn"
+iter_filter_is_ok = "warn"
+iter_filter_is_some = "warn"
 iter_not_returning_iterator = "warn"
 iter_on_empty_collections = "warn"
 iter_on_single_items = "warn"
@@ -427,6 +431,7 @@ macro_use_imports = "warn"
 manual_assert = "warn"
 manual_clamp = "warn"
 manual_instant_elapsed = "warn"
+manual_is_variant_and = "warn"
 manual_let_else = "warn"
 manual_ok_or = "warn"
 manual_string_new = "warn"
@@ -441,6 +446,7 @@ mismatched_target_os = "warn"
 mismatching_type_param_order = "warn"
 missing_enforced_import_renames = "warn"
 missing_safety_doc = "warn"
+mixed_attributes_style = "warn"
 mut_mut = "warn"
 mutex_integer = "warn"
 needless_borrow = "warn"
@@ -450,24 +456,28 @@ needless_pass_by_ref_mut = "warn"
 needless_pass_by_value = "warn"
 negative_feature_names = "warn"
 nonstandard_macro_braces = "warn"
+option_as_ref_cloned = "warn"
 option_option = "warn"
 path_buf_push_overwrite = "warn"
 ptr_as_ptr = "warn"
 ptr_cast_constness = "warn"
+pub_underscore_fields = "warn"
 pub_without_shorthand = "warn"
 rc_mutex = "warn"
 readonly_write_lock = "warn"
 redundant_type_annotations = "warn"
+ref_as_ptr = "warn"
 ref_option_ref = "warn"
 rest_pat_in_fully_bound_structs = "warn"
 same_functions_in_if_condition = "warn"
 semicolon_if_nothing_returned = "warn"
 should_panic_without_expect = "warn"
 significant_drop_tightening = "warn"
 single_match_else = "warn"
+str_split_at_newline = "warn"
 str_to_string = "warn"
-string_add_assign = "warn"
 string_add = "warn"
+string_add_assign = "warn"
 string_lit_as_bytes = "warn"
 string_lit_chars_any = "warn"
 string_to_string = "warn"
@@ -501,6 +511,7 @@ zero_sized_map_values = "warn"
 # Disabled waiting on https://github.com/rust-lang/rust-clippy/issues/9602
 #self_named_module_files = "warn"
 
+assigning_clones = "allow"      # Too much for too little
 manual_range_contains = "allow" # this one is just worse imho
 map_unwrap_or = "allow"         # so is this one
 ref_patterns = "allow"          # It's nice to avoid ref pattern, but there are some situations that are hard (impossible?) to express without.

clippy.toml

@@ -3,7 +3,7 @@
 # -----------------------------------------------------------------------------
 # Section identical to the main scripts/clippy_wasm/clippy.toml:
 
-msrv = "1.76"
+msrv = "1.79"
 
 allow-unwrap-in-tests = true
 
@@ -77,13 +77,16 @@ doc-valid-idents = [
   "GLTF",
   "iOS",
   "macOS",
+  "MessagePack",
+  "MiMalloc",
   "NaN",
   "OBJ",
   "OpenGL",
   "PyPI",
   "sRGB",
   "sRGBA",
   "WebGL",
+  "WebGPU",
   "WebSocket",
   "WebSockets",
 ]

crates/build/re_dev_tools/src/build_examples/example.rs

@@ -129,6 +129,7 @@ impl ExamplesManifest {
 #[derive(serde::Deserialize)]
 pub struct ExampleCategory {
     /// Used to sort categories in the `rerun.io/examples` navbar.
+    #[allow(unused)]
     pub order: u64,
 
     /// `snake_case` name.

crates/build/re_dev_tools/src/build_search_index/util.rs

@@ -20,11 +20,13 @@ pub trait CommandExt {
         I: IntoIterator<Item = S>,
         S: AsRef<OsStr>;
 
+    #[allow(unused)]
     fn with_env<K, V>(self, key: K, val: V) -> Self
     where
         K: AsRef<OsStr>,
         V: AsRef<OsStr>;
 
+    #[allow(unused)]
     fn run(self) -> io::Result<()>;
 
     fn output(self) -> anyhow::Result<Vec<u8>>;

crates/build/re_types_builder/src/codegen/common.rs

@@ -17,7 +17,7 @@ pub struct ExampleInfo<'a> {
     /// Path to the snippet relative to the snippet directory.
     pub path: &'a str,
 
-    /// The snake_case name of the example.
+    /// The `snake_case` name of the example.
     pub name: String,
 
     /// The human-readable name of the example.

crates/build/re_types_builder/src/codegen/docs/mod.rs

@@ -12,6 +12,8 @@ use crate::{
     ATTR_DOCS_VIEW_TYPES,
 };
 
+pub const DATAFRAME_VIEW_FQNAME: &str = "rerun.blueprint.views.DataframeView";
+
 macro_rules! putln {
     ($o:ident) => ( writeln!($o).ok() );
     ($o:ident, $($tt:tt)*) => ( writeln!($o, $($tt)*).ok() );
@@ -574,9 +576,10 @@ fn write_archetype_fields(
         putln!(page, "**Optional**: {}", optional.join(", "));
     }
 
+    putln!(page);
+    putln!(page, "## Shown in");
+
     if let Some(view_types) = view_per_archetype.get(&object.fqname) {
-        putln!(page);
-        putln!(page, "## Shown in");
         for ViewReference {
             view_name,
             explanation,
@@ -592,6 +595,9 @@ fn write_archetype_fields(
             putln!(page);
         }
     }
+
+    // Special case for dataframe view: it can display anything.
+    putln!(page, "* [DataframeView](../views/dataframe_view.md)");
 }
 
 fn write_visualized_archetypes(
@@ -614,7 +620,7 @@ fn write_visualized_archetypes(
         }
     }
 
-    if archetype_fqnames.is_empty() {
+    if archetype_fqnames.is_empty() && view.fqname != DATAFRAME_VIEW_FQNAME {
         reporter.error(&view.virtpath, &view.fqname, "No archetypes use this view.");
         return;
     }
@@ -624,18 +630,24 @@ fn write_visualized_archetypes(
 
     putln!(page, "## Visualized archetypes");
     putln!(page);
-    for (fqname, explanation) in archetype_fqnames {
-        let object = &objects[&fqname];
-        page.push_str(&format!(
-            "* [`{}`](../{}/{}.md)",
-            object.name,
-            object.kind.plural_snake_case(),
-            object.snake_case_name()
-        ));
-        if let Some(explanation) = explanation {
-            page.push_str(&format!(" ({explanation})"));
+
+    // special case for dataframe view
+    if view.fqname == DATAFRAME_VIEW_FQNAME {
+        putln!(page, "Any data can be displayed by the Dataframe view.");
+    } else {
+        for (fqname, explanation) in archetype_fqnames {
+            let object = &objects[&fqname];
+            page.push_str(&format!(
+                "* [`{}`](../{}/{}.md)",
+                object.name,
+                object.kind.plural_snake_case(),
+                object.snake_case_name()
+            ));
+            if let Some(explanation) = explanation {
+                page.push_str(&format!(" ({explanation})"));
+            }
+            putln!(page);
         }
-        putln!(page);
     }
     putln!(page);
 }

crates/build/re_types_builder/src/codegen/python/mod.rs

@@ -185,10 +185,10 @@ struct ExtensionClass {
     /// a default implementation.
     has_array: bool,
 
-    /// Whether the `ObjectExt` contains __native_to_pa_array__()
+    /// Whether the `ObjectExt` contains `__native_to_pa_array__()`
     has_native_to_pa_array: bool,
 
-    /// Whether the `ObjectExt` contains a deferred_patch_class() method
+    /// Whether the `ObjectExt` contains a `deferred_patch_class()` method
     has_deferred_patch_class: bool,
 }
 
@@ -1509,11 +1509,7 @@ fn quote_union_aliases_from_object<'a>(
     let name = &obj.name;
 
     let union_fields = field_types.join(",");
-    let aliases = if let Some(aliases) = aliases {
-        aliases
-    } else {
-        String::new()
-    };
+    let aliases = aliases.unwrap_or_default();
 
     unindent(&format!(
         r#"