From bf74a2ac44bea4fc7e1160c81a2c4d4cc6974b64 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 17:35:36 +0100 Subject: [PATCH 1/9] Start porting fallbacks to arrow-rs --- Cargo.lock | 3 + .../src/codegen/rust/reflection.rs | 2 +- crates/store/re_chunk/src/helpers.rs | 16 +- crates/store/re_chunk_store/tests/gc.rs | 2 +- crates/store/re_chunk_store/tests/reads.rs | 2 +- crates/store/re_query/examples/latest_at.rs | 2 +- crates/store/re_query/src/latest_at.rs | 2 +- crates/store/re_types/src/reflection/mod.rs | 212 +++++++++--------- .../store/re_types_core/src/loggable_batch.rs | 5 + crates/store/re_types_core/src/reflection.rs | 4 +- crates/viewer/re_data_ui/src/component.rs | 2 +- .../re_selection_panel/src/defaults_ui.rs | 5 +- .../re_selection_panel/src/visualizer_ui.rs | 9 +- crates/viewer/re_view/Cargo.toml | 1 + crates/viewer/re_view/src/query.rs | 7 +- crates/viewer/re_view/src/results_ext.rs | 5 +- crates/viewer/re_view/src/view_property_ui.rs | 4 +- .../viewer/re_view_graph/src/ui/selection.rs | 5 +- crates/viewer/re_viewer_context/Cargo.toml | 1 + .../src/blueprint_helpers.rs | 17 +- .../src/component_fallbacks.rs | 14 +- .../src/component_ui_registry.rs | 4 +- .../re_viewer_context/src/viewer_context.rs | 8 +- .../viewer/re_viewport_blueprint/Cargo.toml | 1 + .../src/view_properties.rs | 22 +- 25 files changed, 193 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60076725fe56..0bf98eb72cf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6559,6 +6559,7 @@ dependencies = [ "glam", "itertools 0.13.0", "nohash-hasher", + "re_arrow2", "re_chunk_store", "re_entity_db", "re_log", @@ -6872,6 +6873,7 @@ dependencies = [ "ahash", "anyhow", "arboard", + "arrow", "bit-vec", "bitflags 2.6.0", "bytemuck", @@ -6949,6 +6951,7 @@ name = "re_viewport_blueprint" version = "0.21.0-alpha.1+dev" dependencies = [ "ahash", + "arrow", "egui", "egui_tiles", "itertools 0.13.0", diff --git a/crates/build/re_types_builder/src/codegen/rust/reflection.rs b/crates/build/re_types_builder/src/codegen/rust/reflection.rs index d347ba243cfb..191dde948f92 100644 --- a/crates/build/re_types_builder/src/codegen/rust/reflection.rs +++ b/crates/build/re_types_builder/src/codegen/rust/reflection.rs @@ -137,7 +137,7 @@ fn generate_component_reflection( || contents.contains(&format!("impl Default for super::{}", &obj.name)) }); let custom_placeholder = if auto_derive_default || has_custom_default_impl { - quote! { Some(#type_name::default().to_arrow2()?) } + quote! { Some(#type_name::default().to_arrow()?) } } else { quote! { None } }; diff --git a/crates/store/re_chunk/src/helpers.rs b/crates/store/re_chunk/src/helpers.rs index 1b345c58e560..4e1ac8ddaf5b 100644 --- a/crates/store/re_chunk/src/helpers.rs +++ b/crates/store/re_chunk/src/helpers.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use arrow::array::ArrayRef; use arrow2::array::Array as Arrow2Array; use re_log_types::{TimeInt, Timeline}; @@ -262,7 +263,14 @@ impl UnitChunkShared { /// Returns the raw data for the specified component. #[inline] - pub fn component_batch_raw( + pub fn component_batch_raw(&self, component_name: &ComponentName) -> Option { + self.component_batch_raw_arrow2(component_name) + .map(|array| array.into()) + } + + /// Returns the raw data for the specified component. + #[inline] + pub fn component_batch_raw_arrow2( &self, component_name: &ComponentName, ) -> Option> { @@ -276,7 +284,7 @@ impl UnitChunkShared { /// Returns an error if the data cannot be deserialized. #[inline] pub fn component_batch(&self) -> Option>> { - let data = C::from_arrow2(&*self.component_batch_raw(&C::name())?); + let data = C::from_arrow2(&*self.component_batch_raw_arrow2(&C::name())?); Some(data.map_err(Into::into)) } @@ -291,7 +299,7 @@ impl UnitChunkShared { component_name: &ComponentName, instance_index: usize, ) -> Option>> { - let array = self.component_batch_raw(component_name)?; + let array = self.component_batch_raw_arrow2(component_name)?; if array.len() > instance_index { Some(Ok(array.sliced(instance_index, 1))) } else { @@ -334,7 +342,7 @@ impl UnitChunkShared { &self, component_name: &ComponentName, ) -> Option>> { - let array = self.component_batch_raw(component_name)?; + let array = self.component_batch_raw_arrow2(component_name)?; if array.len() == 1 { Some(Ok(array.sliced(0, 1))) } else { diff --git a/crates/store/re_chunk_store/tests/gc.rs b/crates/store/re_chunk_store/tests/gc.rs index d461fe2ac79e..561ab67e4e30 100644 --- a/crates/store/re_chunk_store/tests/gc.rs +++ b/crates/store/re_chunk_store/tests/gc.rs @@ -37,7 +37,7 @@ fn query_latest_array( }) .max_by_key(|(index, _chunk)| *index)?; - unit.component_batch_raw(&component_name) + unit.component_batch_raw_arrow2(&component_name) .map(|array| (data_time, row_id, array)) } diff --git a/crates/store/re_chunk_store/tests/reads.rs b/crates/store/re_chunk_store/tests/reads.rs index 66b12f3b29ee..f3c48013d7df 100644 --- a/crates/store/re_chunk_store/tests/reads.rs +++ b/crates/store/re_chunk_store/tests/reads.rs @@ -39,7 +39,7 @@ fn query_latest_array( }) .max_by_key(|(index, _chunk)| *index)?; - unit.component_batch_raw(&component_desc.component_name) + unit.component_batch_raw_arrow2(&component_desc.component_name) .map(|array| (data_time, row_id, array)) } diff --git a/crates/store/re_query/examples/latest_at.rs b/crates/store/re_query/examples/latest_at.rs index 7ef6a93abb4c..2cfc8136dbb1 100644 --- a/crates/store/re_query/examples/latest_at.rs +++ b/crates/store/re_query/examples/latest_at.rs @@ -74,7 +74,7 @@ fn main() -> anyhow::Result<()> { // data directly: let colors = colors .context("missing")? - .component_batch_raw(&MyColor::name()) + .component_batch_raw_arrow2(&MyColor::name()) .context("invalid")?; let colors = colors .as_any() diff --git a/crates/store/re_query/src/latest_at.rs b/crates/store/re_query/src/latest_at.rs index a9b61bcaa627..efc84879ae44 100644 --- a/crates/store/re_query/src/latest_at.rs +++ b/crates/store/re_query/src/latest_at.rs @@ -319,7 +319,7 @@ impl LatestAtResults { ) -> Option> { self.components .get(component_name) - .and_then(|unit| unit.component_batch_raw(component_name)) + .and_then(|unit| unit.component_batch_raw_arrow2(component_name)) } /// Returns the deserialized data for the specified component. diff --git a/crates/store/re_types/src/reflection/mod.rs b/crates/store/re_types/src/reflection/mod.rs index 05839fe7c80e..5555c742eceb 100644 --- a/crates/store/re_types/src/reflection/mod.rs +++ b/crates/store/re_types/src/reflection/mod.rs @@ -37,7 +37,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The active tab in a tabbed container.", - custom_placeholder: Some(ActiveTab::default().to_arrow2()?), + custom_placeholder: Some(ActiveTab::default().to_arrow()?), datatype: ActiveTab::arrow2_datatype(), }, ), @@ -45,7 +45,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Whether empty cells in a dataframe should be filled with a latest-at query.", - custom_placeholder: Some(ApplyLatestAt::default().to_arrow2()?), + custom_placeholder: Some(ApplyLatestAt::default().to_arrow()?), datatype: ApplyLatestAt::arrow2_datatype(), }, ), @@ -53,7 +53,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Whether the viewport layout is determined automatically.", - custom_placeholder: Some(AutoLayout::default().to_arrow2()?), + custom_placeholder: Some(AutoLayout::default().to_arrow()?), datatype: AutoLayout::arrow2_datatype(), }, ), @@ -61,7 +61,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Whether or not views should be created automatically.", - custom_placeholder: Some(AutoViews::default().to_arrow2()?), + custom_placeholder: Some(AutoViews::default().to_arrow()?), datatype: AutoViews::arrow2_datatype(), }, ), @@ -69,7 +69,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The type of the background in a view.", - custom_placeholder: Some(BackgroundKind::default().to_arrow2()?), + custom_placeholder: Some(BackgroundKind::default().to_arrow()?), datatype: BackgroundKind::arrow2_datatype(), }, ), @@ -77,7 +77,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The layout share of a column in the container.", - custom_placeholder: Some(ColumnShare::default().to_arrow2()?), + custom_placeholder: Some(ColumnShare::default().to_arrow()?), datatype: ColumnShare::arrow2_datatype(), }, ), @@ -85,9 +85,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Describe a component column to be selected in the dataframe view.", - custom_placeholder: Some( - ComponentColumnSelector::default().to_arrow2()?, - ), + custom_placeholder: Some(ComponentColumnSelector::default().to_arrow()?), datatype: ComponentColumnSelector::arrow2_datatype(), }, ), @@ -95,7 +93,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The kind of a blueprint container (tabs, grid, …).", - custom_placeholder: Some(ContainerKind::default().to_arrow2()?), + custom_placeholder: Some(ContainerKind::default().to_arrow()?), datatype: ContainerKind::arrow2_datatype(), }, ), @@ -103,7 +101,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "One of four 2D corners, typically used to align objects.", - custom_placeholder: Some(Corner2D::default().to_arrow2()?), + custom_placeholder: Some(Corner2D::default().to_arrow()?), datatype: Corner2D::arrow2_datatype(), }, ), @@ -111,7 +109,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Whether a procedure is enabled.", - custom_placeholder: Some(Enabled::default().to_arrow2()?), + custom_placeholder: Some(Enabled::default().to_arrow()?), datatype: Enabled::arrow2_datatype(), }, ), @@ -119,7 +117,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Configuration for a filter-by-range feature of the dataframe view.", - custom_placeholder: Some(FilterByRange::default().to_arrow2()?), + custom_placeholder: Some(FilterByRange::default().to_arrow()?), datatype: FilterByRange::arrow2_datatype(), }, ), @@ -127,7 +125,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Configuration for the filter is not null feature of the dataframe view.", - custom_placeholder: Some(FilterIsNotNull::default().to_arrow2()?), + custom_placeholder: Some(FilterIsNotNull::default().to_arrow()?), datatype: FilterIsNotNull::arrow2_datatype(), }, ), @@ -135,7 +133,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The target distance between two nodes.\n\nThis is helpful to scale the layout, for example if long labels are involved.", - custom_placeholder: Some(ForceDistance::default().to_arrow2()?), + custom_placeholder: Some(ForceDistance::default().to_arrow()?), datatype: ForceDistance::arrow2_datatype(), }, ), @@ -143,7 +141,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Specifies how often this force should be applied per iteration.\n\nIncreasing this parameter can lead to better results at the cost of longer computation time.", - custom_placeholder: Some(ForceIterations::default().to_arrow2()?), + custom_placeholder: Some(ForceIterations::default().to_arrow()?), datatype: ForceIterations::arrow2_datatype(), }, ), @@ -151,7 +149,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The strength of a given force.\n\nAllows to assign different weights to the individual forces, prioritizing one over the other.", - custom_placeholder: Some(ForceStrength::default().to_arrow2()?), + custom_placeholder: Some(ForceStrength::default().to_arrow()?), datatype: ForceStrength::arrow2_datatype(), }, ), @@ -159,7 +157,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "How many columns a grid container should have.", - custom_placeholder: Some(GridColumns::default().to_arrow2()?), + custom_placeholder: Some(GridColumns::default().to_arrow()?), datatype: GridColumns::arrow2_datatype(), }, ), @@ -167,7 +165,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Space between grid lines of one line to the next in scene units.", - custom_placeholder: Some(GridSpacing::default().to_arrow2()?), + custom_placeholder: Some(GridSpacing::default().to_arrow()?), datatype: GridSpacing::arrow2_datatype(), }, ), @@ -175,7 +173,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "All the contents in the container.", - custom_placeholder: Some(IncludedContent::default().to_arrow2()?), + custom_placeholder: Some(IncludedContent::default().to_arrow()?), datatype: IncludedContent::arrow2_datatype(), }, ), @@ -183,7 +181,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Whether the entity can be interacted with.\n\nNon interactive components are still visible, but mouse interactions in the view are disabled.", - custom_placeholder: Some(Interactive::default().to_arrow2()?), + custom_placeholder: Some(Interactive::default().to_arrow()?), datatype: Interactive::arrow2_datatype(), }, ), @@ -191,7 +189,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Indicate whether the range should be locked when zooming in on the data.\n\nDefault is `false`, i.e. zoom will change the visualized range.", - custom_placeholder: Some(LockRangeDuringZoom::default().to_arrow2()?), + custom_placeholder: Some(LockRangeDuringZoom::default().to_arrow()?), datatype: LockRangeDuringZoom::arrow2_datatype(), }, ), @@ -199,7 +197,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Name of the map provider to be used in Map views.", - custom_placeholder: Some(MapProvider::default().to_arrow2()?), + custom_placeholder: Some(MapProvider::default().to_arrow()?), datatype: MapProvider::arrow2_datatype(), }, ), @@ -207,7 +205,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Distance to the near clip plane used for `Spatial2DView`.", - custom_placeholder: Some(NearClipPlane::default().to_arrow2()?), + custom_placeholder: Some(NearClipPlane::default().to_arrow()?), datatype: NearClipPlane::arrow2_datatype(), }, ), @@ -215,7 +213,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Tri-state for panel controls.", - custom_placeholder: Some(PanelState::default().to_arrow2()?), + custom_placeholder: Some(PanelState::default().to_arrow()?), datatype: PanelState::arrow2_datatype(), }, ), @@ -223,7 +221,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "An individual query expression used to filter a set of [`datatypes.EntityPath`](https://rerun.io/docs/reference/types/datatypes/entity_path)s.\n\nEach expression is either an inclusion or an exclusion expression.\nInclusions start with an optional `+` and exclusions must start with a `-`.\n\nMultiple expressions are combined together as part of archetypes.ViewContents.\n\nThe `/**` suffix matches the whole subtree, i.e. self and any child, recursively\n(`/world/**` matches both `/world` and `/world/car/driver`).\nOther uses of `*` are not (yet) supported.", - custom_placeholder: Some(QueryExpression::default().to_arrow2()?), + custom_placeholder: Some(QueryExpression::default().to_arrow()?), datatype: QueryExpression::arrow2_datatype(), }, ), @@ -231,7 +229,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The container that sits at the root of a viewport.", - custom_placeholder: Some(RootContainer::default().to_arrow2()?), + custom_placeholder: Some(RootContainer::default().to_arrow()?), datatype: RootContainer::arrow2_datatype(), }, ), @@ -239,7 +237,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The layout share of a row in the container.", - custom_placeholder: Some(RowShare::default().to_arrow2()?), + custom_placeholder: Some(RowShare::default().to_arrow()?), datatype: RowShare::arrow2_datatype(), }, ), @@ -247,7 +245,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Describe a component column to be selected in the dataframe view.", - custom_placeholder: Some(SelectedColumns::default().to_arrow2()?), + custom_placeholder: Some(SelectedColumns::default().to_arrow()?), datatype: SelectedColumns::arrow2_datatype(), }, ), @@ -256,7 +254,7 @@ fn generate_component_reflection() -> Result Result::name(), ComponentReflection { docstring_md: "A timeline identified by its name.", - custom_placeholder: Some(TimelineName::default().to_arrow2()?), + custom_placeholder: Some(TimelineName::default().to_arrow()?), datatype: TimelineName::arrow2_datatype(), }, ), @@ -273,7 +271,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The class identifier of view, e.g. `\"2D\"`, `\"TextLog\"`, ….", - custom_placeholder: Some(ViewClass::default().to_arrow2()?), + custom_placeholder: Some(ViewClass::default().to_arrow()?), datatype: ViewClass::arrow2_datatype(), }, ), @@ -281,7 +279,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Determines whether an image or texture should be scaled to fit the viewport.", - custom_placeholder: Some(ViewFit::default().to_arrow2()?), + custom_placeholder: Some(ViewFit::default().to_arrow()?), datatype: ViewFit::arrow2_datatype(), }, ), @@ -289,7 +287,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Whether a view is maximized.", - custom_placeholder: Some(ViewMaximized::default().to_arrow2()?), + custom_placeholder: Some(ViewMaximized::default().to_arrow()?), datatype: ViewMaximized::arrow2_datatype(), }, ), @@ -297,7 +295,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The origin of a view.", - custom_placeholder: Some(ViewOrigin::default().to_arrow2()?), + custom_placeholder: Some(ViewOrigin::default().to_arrow()?), datatype: ViewOrigin::arrow2_datatype(), }, ), @@ -306,7 +304,7 @@ fn generate_component_reflection() -> Result Result::name(), ComponentReflection { docstring_md: "Whether the container, view, entity or instance is currently visible.", - custom_placeholder: Some(Visible::default().to_arrow2()?), + custom_placeholder: Some(Visible::default().to_arrow()?), datatype: Visible::arrow2_datatype(), }, ), @@ -323,7 +321,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The range of values on a given timeline that will be included in a view's query.\n\nRefer to `VisibleTimeRanges` archetype for more information.", - custom_placeholder: Some(VisibleTimeRange::default().to_arrow2()?), + custom_placeholder: Some(VisibleTimeRange::default().to_arrow()?), datatype: VisibleTimeRange::arrow2_datatype(), }, ), @@ -331,7 +329,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Visual bounds in 2D space used for `Spatial2DView`.", - custom_placeholder: Some(VisualBounds2D::default().to_arrow2()?), + custom_placeholder: Some(VisualBounds2D::default().to_arrow()?), datatype: VisualBounds2D::arrow2_datatype(), }, ), @@ -339,7 +337,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Override the visualizers for an entity.\n\nThis component is a stop-gap mechanism based on the current implementation details\nof the visualizer system. It is not intended to be a long-term solution, but provides\nenough utility to be useful in the short term.\n\nThe long-term solution is likely to be based off: \n\nThis can only be used as part of blueprints. It will have no effect if used\nin a regular entity.", - custom_placeholder: Some(VisualizerOverrides::default().to_arrow2()?), + custom_placeholder: Some(VisualizerOverrides::default().to_arrow()?), datatype: VisualizerOverrides::arrow2_datatype(), }, ), @@ -347,7 +345,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A zoom level determines how much of the world is visible on a map.", - custom_placeholder: Some(ZoomLevel::default().to_arrow2()?), + custom_placeholder: Some(ZoomLevel::default().to_arrow()?), datatype: ZoomLevel::arrow2_datatype(), }, ), @@ -355,7 +353,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Policy for aggregation of multiple scalar plot values.\n\nThis is used for lines in plots when the X axis distance of individual points goes below a single pixel,\ni.e. a single pixel covers more than one tick worth of data. It can greatly improve performance\n(and readability) in such situations as it prevents overdraw.", - custom_placeholder: Some(AggregationPolicy::default().to_arrow2()?), + custom_placeholder: Some(AggregationPolicy::default().to_arrow()?), datatype: AggregationPolicy::arrow2_datatype(), }, ), @@ -363,7 +361,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A color multiplier, usually applied to a whole entity, e.g. a mesh.", - custom_placeholder: Some(AlbedoFactor::default().to_arrow2()?), + custom_placeholder: Some(AlbedoFactor::default().to_arrow()?), datatype: AlbedoFactor::arrow2_datatype(), }, ), @@ -371,7 +369,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The annotation context provides additional information on how to display entities.\n\nEntities can use [`datatypes.ClassId`](https://rerun.io/docs/reference/types/datatypes/class_id)s and [`datatypes.KeypointId`](https://rerun.io/docs/reference/types/datatypes/keypoint_id)s to provide annotations, and\nthe labels and colors will be looked up in the appropriate\nannotation context. We use the *first* annotation context we find in the\npath-hierarchy when searching up through the ancestors of a given entity\npath.", - custom_placeholder: Some(AnnotationContext::default().to_arrow2()?), + custom_placeholder: Some(AnnotationContext::default().to_arrow()?), datatype: AnnotationContext::arrow2_datatype(), }, ), @@ -379,7 +377,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The length of an axis in local units of the space.", - custom_placeholder: Some(AxisLength::default().to_arrow2()?), + custom_placeholder: Some(AxisLength::default().to_arrow()?), datatype: AxisLength::arrow2_datatype(), }, ), @@ -395,7 +393,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 16-bit ID representing a type of semantic class.", - custom_placeholder: Some(ClassId::default().to_arrow2()?), + custom_placeholder: Some(ClassId::default().to_arrow()?), datatype: ClassId::arrow2_datatype(), }, ), @@ -403,7 +401,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Configures how a clear operation should behave - recursive or not.", - custom_placeholder: Some(ClearIsRecursive::default().to_arrow2()?), + custom_placeholder: Some(ClearIsRecursive::default().to_arrow()?), datatype: ClearIsRecursive::arrow2_datatype(), }, ), @@ -411,7 +409,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "An RGBA color with unmultiplied/separate alpha, in sRGB gamma space with linear alpha.\n\nThe color is stored as a 32-bit integer, where the most significant\nbyte is `R` and the least significant byte is `A`.", - custom_placeholder: Some(Color::default().to_arrow2()?), + custom_placeholder: Some(Color::default().to_arrow()?), datatype: Color::arrow2_datatype(), }, ), @@ -419,7 +417,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Colormap for mapping scalar values within a given range to a color.\n\nThis provides a number of popular pre-defined colormaps.\nIn the future, the Rerun Viewer will allow users to define their own colormaps,\nbut currently the Viewer is limited to the types defined here.", - custom_placeholder: Some(Colormap::default().to_arrow2()?), + custom_placeholder: Some(Colormap::default().to_arrow()?), datatype: Colormap::arrow2_datatype(), }, ), @@ -427,7 +425,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The world->depth map scaling factor.\n\nThis measures how many depth map units are in a world unit.\nFor instance, if a depth map uses millimeters and the world uses meters,\nthis value would be `1000`.\n\nNote that the only effect on 2D views is the physical depth values shown when hovering the image.\nIn 3D views on the other hand, this affects where the points of the point cloud are placed.", - custom_placeholder: Some(DepthMeter::default().to_arrow2()?), + custom_placeholder: Some(DepthMeter::default().to_arrow()?), datatype: DepthMeter::arrow2_datatype(), }, ), @@ -435,7 +433,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Spatially disconnect this entity from its parent.\n\nSpecifies that the entity path at which this is logged is spatially disconnected from its parent,\nmaking it impossible to transform the entity path into its parent's space and vice versa.\nIt *only* applies to views that work with spatial transformations, i.e. 2D & 3D views.\nThis is useful for specifying that a subgraph is independent of the rest of the scene.", - custom_placeholder: Some(DisconnectedSpace::default().to_arrow2()?), + custom_placeholder: Some(DisconnectedSpace::default().to_arrow()?), datatype: DisconnectedSpace::arrow2_datatype(), }, ), @@ -443,7 +441,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Draw order of 2D elements. Higher values are drawn on top of lower values.\n\nAn entity can have only a single draw order component.\nWithin an entity draw order is governed by the order of the components.\n\nDraw order for entities with the same draw order is generally undefined.", - custom_placeholder: Some(DrawOrder::default().to_arrow2()?), + custom_placeholder: Some(DrawOrder::default().to_arrow()?), datatype: DrawOrder::arrow2_datatype(), }, ), @@ -451,7 +449,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A path to an entity, usually to reference some data that is part of the target entity.", - custom_placeholder: Some(EntityPath::default().to_arrow2()?), + custom_placeholder: Some(EntityPath::default().to_arrow()?), datatype: EntityPath::arrow2_datatype(), }, ), @@ -459,7 +457,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "How a geometric shape is drawn and colored.", - custom_placeholder: Some(FillMode::default().to_arrow2()?), + custom_placeholder: Some(FillMode::default().to_arrow()?), datatype: FillMode::arrow2_datatype(), }, ), @@ -467,7 +465,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "How much a primitive fills out the available space.\n\nUsed for instance to scale the points of the point cloud created from [`archetypes.DepthImage`](https://rerun.io/docs/reference/types/archetypes/depth_image) projection in 3D views.\nValid range is from 0 to max float although typically values above 1.0 are not useful.\n\nDefaults to 1.0.", - custom_placeholder: Some(FillRatio::default().to_arrow2()?), + custom_placeholder: Some(FillRatio::default().to_arrow()?), datatype: FillRatio::arrow2_datatype(), }, ), @@ -475,7 +473,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A gamma correction value to be used with a scalar value or color.\n\nUsed to adjust the gamma of a color or scalar value between 0 and 1 before rendering.\n`new_value = old_value ^ gamma`\n\nValid range is from 0 (excluding) to max float.\nDefaults to 1.0 unless otherwise specified.", - custom_placeholder: Some(GammaCorrection::default().to_arrow2()?), + custom_placeholder: Some(GammaCorrection::default().to_arrow()?), datatype: GammaCorrection::arrow2_datatype(), }, ), @@ -483,7 +481,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A geospatial line string expressed in [EPSG:4326](https://epsg.io/4326) latitude and longitude (North/East-positive degrees).", - custom_placeholder: Some(GeoLineString::default().to_arrow2()?), + custom_placeholder: Some(GeoLineString::default().to_arrow()?), datatype: GeoLineString::arrow2_datatype(), }, ), @@ -491,7 +489,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "An edge in a graph connecting two nodes.", - custom_placeholder: Some(GraphEdge::default().to_arrow2()?), + custom_placeholder: Some(GraphEdge::default().to_arrow()?), datatype: GraphEdge::arrow2_datatype(), }, ), @@ -499,7 +497,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A string-based ID representing a node in a graph.", - custom_placeholder: Some(GraphNode::default().to_arrow2()?), + custom_placeholder: Some(GraphNode::default().to_arrow()?), datatype: GraphNode::arrow2_datatype(), }, ), @@ -507,7 +505,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Specifies if a graph has directed or undirected edges.", - custom_placeholder: Some(GraphType::default().to_arrow2()?), + custom_placeholder: Some(GraphType::default().to_arrow()?), datatype: GraphType::arrow2_datatype(), }, ), @@ -515,7 +513,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Half-size (radius) of a 2D box.\n\nMeasured in its local coordinate system.\n\nThe box extends both in negative and positive direction along each axis.\nNegative sizes indicate that the box is flipped along the respective axis, but this has no effect on how it is displayed.", - custom_placeholder: Some(HalfSize2D::default().to_arrow2()?), + custom_placeholder: Some(HalfSize2D::default().to_arrow()?), datatype: HalfSize2D::arrow2_datatype(), }, ), @@ -523,7 +521,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Half-size (radius) of a 3D box.\n\nMeasured in its local coordinate system.\n\nThe box extends both in negative and positive direction along each axis.\nNegative sizes indicate that the box is flipped along the respective axis, but this has no effect on how it is displayed.", - custom_placeholder: Some(HalfSize3D::default().to_arrow2()?), + custom_placeholder: Some(HalfSize3D::default().to_arrow()?), datatype: HalfSize3D::arrow2_datatype(), }, ), @@ -539,7 +537,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The metadata describing the contents of a [`components.ImageBuffer`](https://rerun.io/docs/reference/types/components/image_buffer).", - custom_placeholder: Some(ImageFormat::default().to_arrow2()?), + custom_placeholder: Some(ImageFormat::default().to_arrow()?), datatype: ImageFormat::arrow2_datatype(), }, ), @@ -547,7 +545,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The distance from the camera origin to the image plane when the projection is shown in a 3D viewer.\n\nThis is only used for visualization purposes, and does not affect the projection itself.", - custom_placeholder: Some(ImagePlaneDistance::default().to_arrow2()?), + custom_placeholder: Some(ImagePlaneDistance::default().to_arrow()?), datatype: ImagePlaneDistance::arrow2_datatype(), }, ), @@ -555,7 +553,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 16-bit ID representing a type of semantic keypoint within a class.", - custom_placeholder: Some(KeypointId::default().to_arrow2()?), + custom_placeholder: Some(KeypointId::default().to_arrow()?), datatype: KeypointId::arrow2_datatype(), }, ), @@ -563,7 +561,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A geospatial position expressed in [EPSG:4326](https://epsg.io/4326) latitude and longitude (North/East-positive degrees).", - custom_placeholder: Some(LatLon::default().to_arrow2()?), + custom_placeholder: Some(LatLon::default().to_arrow()?), datatype: LatLon::arrow2_datatype(), }, ), @@ -571,7 +569,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Length, or one-dimensional size.\n\nMeasured in its local coordinate system; consult the archetype in use to determine which\naxis or part of the entity this is the length of.", - custom_placeholder: Some(Length::default().to_arrow2()?), + custom_placeholder: Some(Length::default().to_arrow()?), datatype: Length::arrow2_datatype(), }, ), @@ -579,7 +577,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A line strip in 2D space.\n\nA line strip is a list of points connected by line segments. It can be used to draw\napproximations of smooth curves.\n\nThe points will be connected in order, like so:\n```text\n 2------3 5\n / \\ /\n0----1 \\ /\n 4\n```", - custom_placeholder: Some(LineStrip2D::default().to_arrow2()?), + custom_placeholder: Some(LineStrip2D::default().to_arrow()?), datatype: LineStrip2D::arrow2_datatype(), }, ), @@ -587,7 +585,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A line strip in 3D space.\n\nA line strip is a list of points connected by line segments. It can be used to draw\napproximations of smooth curves.\n\nThe points will be connected in order, like so:\n```text\n 2------3 5\n / \\ /\n0----1 \\ /\n 4\n```", - custom_placeholder: Some(LineStrip3D::default().to_arrow2()?), + custom_placeholder: Some(LineStrip3D::default().to_arrow()?), datatype: LineStrip3D::arrow2_datatype(), }, ), @@ -595,7 +593,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Filter used when magnifying an image/texture such that a single pixel/texel is displayed as multiple pixels on screen.", - custom_placeholder: Some(MagnificationFilter::default().to_arrow2()?), + custom_placeholder: Some(MagnificationFilter::default().to_arrow()?), datatype: MagnificationFilter::arrow2_datatype(), }, ), @@ -603,7 +601,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The visual appearance of a point in e.g. a 2D plot.", - custom_placeholder: Some(MarkerShape::default().to_arrow2()?), + custom_placeholder: Some(MarkerShape::default().to_arrow()?), datatype: MarkerShape::arrow2_datatype(), }, ), @@ -611,7 +609,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Radius of a marker of a point in e.g. a 2D plot, measured in UI points.", - custom_placeholder: Some(MarkerSize::default().to_arrow2()?), + custom_placeholder: Some(MarkerSize::default().to_arrow()?), datatype: MarkerSize::arrow2_datatype(), }, ), @@ -619,7 +617,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A standardized media type (RFC2046, formerly known as MIME types), encoded as a string.\n\nThe complete reference of officially registered media types is maintained by the IANA and can be\nconsulted at .", - custom_placeholder: Some(MediaType::default().to_arrow2()?), + custom_placeholder: Some(MediaType::default().to_arrow()?), datatype: MediaType::arrow2_datatype(), }, ), @@ -627,7 +625,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A display name, typically for an entity or a item like a plot series.", - custom_placeholder: Some(Name::default().to_arrow2()?), + custom_placeholder: Some(Name::default().to_arrow()?), datatype: Name::arrow2_datatype(), }, ), @@ -635,7 +633,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Degree of transparency ranging from 0.0 (fully transparent) to 1.0 (fully opaque).\n\nThe final opacity value may be a result of multiplication with alpha values as specified by other color sources.\nUnless otherwise specified, the default value is 1.", - custom_placeholder: Some(Opacity::default().to_arrow2()?), + custom_placeholder: Some(Opacity::default().to_arrow()?), datatype: Opacity::arrow2_datatype(), }, ), @@ -643,7 +641,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Camera projection, from image coordinates to view coordinates.\n\nChild from parent.\nImage coordinates from camera view coordinates.\n\nExample:\n```text\n1496.1 0.0 980.5\n 0.0 1496.1 744.5\n 0.0 0.0 1.0\n```", - custom_placeholder: Some(PinholeProjection::default().to_arrow2()?), + custom_placeholder: Some(PinholeProjection::default().to_arrow()?), datatype: PinholeProjection::arrow2_datatype(), }, ), @@ -659,7 +657,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "3D rotation represented by a rotation around a given axis that doesn't propagate in the transform hierarchy.\n\nIf normalization of the rotation axis fails the rotation is treated as an invalid transform.", - custom_placeholder: Some(PoseRotationAxisAngle::default().to_arrow2()?), + custom_placeholder: Some(PoseRotationAxisAngle::default().to_arrow()?), datatype: PoseRotationAxisAngle::arrow2_datatype(), }, ), @@ -667,7 +665,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 3D rotation expressed as a quaternion that doesn't propagate in the transform hierarchy.\n\nNote: although the x,y,z,w components of the quaternion will be passed through to the\ndatastore as provided, when used in the Viewer, quaternions will always be normalized.\nIf normalization fails the rotation is treated as an invalid transform.", - custom_placeholder: Some(PoseRotationQuat::default().to_arrow2()?), + custom_placeholder: Some(PoseRotationQuat::default().to_arrow()?), datatype: PoseRotationQuat::arrow2_datatype(), }, ), @@ -675,7 +673,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 3D scale factor that doesn't propagate in the transform hierarchy.\n\nA scale of 1.0 means no scaling.\nA scale of 2.0 means doubling the size.\nEach component scales along the corresponding axis.", - custom_placeholder: Some(PoseScale3D::default().to_arrow2()?), + custom_placeholder: Some(PoseScale3D::default().to_arrow()?), datatype: PoseScale3D::arrow2_datatype(), }, ), @@ -683,7 +681,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 3x3 transformation matrix Matrix that doesn't propagate in the transform hierarchy.\n\n3x3 matrixes are able to represent any affine transformation in 3D space,\ni.e. rotation, scaling, shearing, reflection etc.\n\nMatrices in Rerun are stored as flat list of coefficients in column-major order:\n```text\n column 0 column 1 column 2\n -------------------------------------------------\nrow 0 | flat_columns[0] flat_columns[3] flat_columns[6]\nrow 1 | flat_columns[1] flat_columns[4] flat_columns[7]\nrow 2 | flat_columns[2] flat_columns[5] flat_columns[8]\n```", - custom_placeholder: Some(PoseTransformMat3x3::default().to_arrow2()?), + custom_placeholder: Some(PoseTransformMat3x3::default().to_arrow()?), datatype: PoseTransformMat3x3::arrow2_datatype(), }, ), @@ -691,7 +689,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A translation vector in 3D space that doesn't propagate in the transform hierarchy.", - custom_placeholder: Some(PoseTranslation3D::default().to_arrow2()?), + custom_placeholder: Some(PoseTranslation3D::default().to_arrow()?), datatype: PoseTranslation3D::arrow2_datatype(), }, ), @@ -699,7 +697,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A position in 2D space.", - custom_placeholder: Some(Position2D::default().to_arrow2()?), + custom_placeholder: Some(Position2D::default().to_arrow()?), datatype: Position2D::arrow2_datatype(), }, ), @@ -707,7 +705,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A position in 3D space.", - custom_placeholder: Some(Position3D::default().to_arrow2()?), + custom_placeholder: Some(Position3D::default().to_arrow()?), datatype: Position3D::arrow2_datatype(), }, ), @@ -715,7 +713,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The radius of something, e.g. a point.\n\nInternally, positive values indicate scene units, whereas negative values\nare interpreted as UI points.\n\nUI points are independent of zooming in Views, but are sensitive to the application UI scaling.\nat 100% UI scaling, UI points are equal to pixels\nThe Viewer's UI scaling defaults to the OS scaling which typically is 100% for full HD screens and 200% for 4k screens.", - custom_placeholder: Some(Radius::default().to_arrow2()?), + custom_placeholder: Some(Radius::default().to_arrow()?), datatype: Radius::arrow2_datatype(), }, ), @@ -723,7 +721,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 1D range, specifying a lower and upper bound.", - custom_placeholder: Some(Range1D::default().to_arrow2()?), + custom_placeholder: Some(Range1D::default().to_arrow()?), datatype: Range1D::arrow2_datatype(), }, ), @@ -739,7 +737,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Pixel resolution width & height, e.g. of a camera sensor.\n\nTypically in integer units, but for some use cases floating point may be used.", - custom_placeholder: Some(Resolution::default().to_arrow2()?), + custom_placeholder: Some(Resolution::default().to_arrow()?), datatype: Resolution::arrow2_datatype(), }, ), @@ -747,7 +745,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "3D rotation represented by a rotation around a given axis.\n\nIf normalization of the rotation axis fails the rotation is treated as an invalid transform.", - custom_placeholder: Some(RotationAxisAngle::default().to_arrow2()?), + custom_placeholder: Some(RotationAxisAngle::default().to_arrow()?), datatype: RotationAxisAngle::arrow2_datatype(), }, ), @@ -755,7 +753,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 3D rotation expressed as a quaternion.\n\nNote: although the x,y,z,w components of the quaternion will be passed through to the\ndatastore as provided, when used in the Viewer, quaternions will always be normalized.\nIf normalization fails the rotation is treated as an invalid transform.", - custom_placeholder: Some(RotationQuat::default().to_arrow2()?), + custom_placeholder: Some(RotationQuat::default().to_arrow()?), datatype: RotationQuat::arrow2_datatype(), }, ), @@ -763,7 +761,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A scalar value, encoded as a 64-bit floating point.\n\nUsed for time series plots.", - custom_placeholder: Some(Scalar::default().to_arrow2()?), + custom_placeholder: Some(Scalar::default().to_arrow()?), datatype: Scalar::arrow2_datatype(), }, ), @@ -771,7 +769,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 3D scale factor.\n\nA scale of 1.0 means no scaling.\nA scale of 2.0 means doubling the size.\nEach component scales along the corresponding axis.", - custom_placeholder: Some(Scale3D::default().to_arrow2()?), + custom_placeholder: Some(Scale3D::default().to_arrow()?), datatype: Scale3D::arrow2_datatype(), }, ), @@ -787,7 +785,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The width of a stroke specified in UI points.", - custom_placeholder: Some(StrokeWidth::default().to_arrow2()?), + custom_placeholder: Some(StrokeWidth::default().to_arrow()?), datatype: StrokeWidth::arrow2_datatype(), }, ), @@ -795,7 +793,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "An N-dimensional array of numbers.\n\nThe number of dimensions and their respective lengths is specified by the `shape` field.\nThe dimensions are ordered from outermost to innermost. For example, in the common case of\na 2D RGB Image, the shape would be `[height, width, channel]`.\n\nThese dimensions are combined with an index to look up values from the `buffer` field,\nwhich stores a contiguous array of typed values.", - custom_placeholder: Some(TensorData::default().to_arrow2()?), + custom_placeholder: Some(TensorData::default().to_arrow()?), datatype: TensorData::arrow2_datatype(), }, ), @@ -804,7 +802,7 @@ fn generate_component_reflection() -> Result Result::name(), ComponentReflection { docstring_md: "Specifies which dimension to use for height.", - custom_placeholder: Some(TensorHeightDimension::default().to_arrow2()?), + custom_placeholder: Some(TensorHeightDimension::default().to_arrow()?), datatype: TensorHeightDimension::arrow2_datatype(), }, ), @@ -821,7 +819,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Specifies which dimension to use for width.", - custom_placeholder: Some(TensorWidthDimension::default().to_arrow2()?), + custom_placeholder: Some(TensorWidthDimension::default().to_arrow()?), datatype: TensorWidthDimension::arrow2_datatype(), }, ), @@ -829,7 +827,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 2D texture UV coordinate.\n\nTexture coordinates specify a position on a 2D texture.\nA range from 0-1 covers the entire texture in the respective dimension.\nUnless configured otherwise, the texture repeats outside of this range.\nRerun uses top-left as the origin for UV coordinates.\n\n 0 U 1\n0 + --------- →\n | .\nV | .\n | .\n1 ↓ . . . . . .\n\nThis is the same convention as in Vulkan/Metal/DX12/WebGPU, but (!) unlike OpenGL,\nwhich places the origin at the bottom-left.", - custom_placeholder: Some(Texcoord2D::default().to_arrow2()?), + custom_placeholder: Some(Texcoord2D::default().to_arrow()?), datatype: Texcoord2D::arrow2_datatype(), }, ), @@ -837,7 +835,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A string of text, e.g. for labels and text documents.", - custom_placeholder: Some(Text::default().to_arrow2()?), + custom_placeholder: Some(Text::default().to_arrow()?), datatype: Text::arrow2_datatype(), }, ), @@ -845,7 +843,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The severity level of a text log message.\n\nRecommended to be one of:\n* `\"CRITICAL\"`\n* `\"ERROR\"`\n* `\"WARN\"`\n* `\"INFO\"`\n* `\"DEBUG\"`\n* `\"TRACE\"`", - custom_placeholder: Some(TextLogLevel::default().to_arrow2()?), + custom_placeholder: Some(TextLogLevel::default().to_arrow()?), datatype: TextLogLevel::arrow2_datatype(), }, ), @@ -853,7 +851,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A 3x3 transformation matrix Matrix.\n\n3x3 matrixes are able to represent any affine transformation in 3D space,\ni.e. rotation, scaling, shearing, reflection etc.\n\nMatrices in Rerun are stored as flat list of coefficients in column-major order:\n```text\n column 0 column 1 column 2\n -------------------------------------------------\nrow 0 | flat_columns[0] flat_columns[3] flat_columns[6]\nrow 1 | flat_columns[1] flat_columns[4] flat_columns[7]\nrow 2 | flat_columns[2] flat_columns[5] flat_columns[8]\n```", - custom_placeholder: Some(TransformMat3x3::default().to_arrow2()?), + custom_placeholder: Some(TransformMat3x3::default().to_arrow()?), datatype: TransformMat3x3::arrow2_datatype(), }, ), @@ -861,7 +859,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Specifies relation a spatial transform describes.", - custom_placeholder: Some(TransformRelation::default().to_arrow2()?), + custom_placeholder: Some(TransformRelation::default().to_arrow()?), datatype: TransformRelation::arrow2_datatype(), }, ), @@ -869,7 +867,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A translation vector in 3D space.", - custom_placeholder: Some(Translation3D::default().to_arrow2()?), + custom_placeholder: Some(Translation3D::default().to_arrow()?), datatype: Translation3D::arrow2_datatype(), }, ), @@ -877,7 +875,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "The three indices of a triangle in a triangle mesh.", - custom_placeholder: Some(TriangleIndices::default().to_arrow2()?), + custom_placeholder: Some(TriangleIndices::default().to_arrow()?), datatype: TriangleIndices::arrow2_datatype(), }, ), @@ -885,7 +883,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Range of expected or valid values, specifying a lower and upper bound.", - custom_placeholder: Some(ValueRange::default().to_arrow2()?), + custom_placeholder: Some(ValueRange::default().to_arrow()?), datatype: ValueRange::arrow2_datatype(), }, ), @@ -893,7 +891,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A vector in 2D space.", - custom_placeholder: Some(Vector2D::default().to_arrow2()?), + custom_placeholder: Some(Vector2D::default().to_arrow()?), datatype: Vector2D::arrow2_datatype(), }, ), @@ -901,7 +899,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "A vector in 3D space.", - custom_placeholder: Some(Vector3D::default().to_arrow2()?), + custom_placeholder: Some(Vector3D::default().to_arrow()?), datatype: Vector3D::arrow2_datatype(), }, ), @@ -909,7 +907,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "Timestamp inside a [`archetypes.AssetVideo`](https://rerun.io/docs/reference/types/archetypes/asset_video).", - custom_placeholder: Some(VideoTimestamp::default().to_arrow2()?), + custom_placeholder: Some(VideoTimestamp::default().to_arrow()?), datatype: VideoTimestamp::arrow2_datatype(), }, ), @@ -917,7 +915,7 @@ fn generate_component_reflection() -> Result::name(), ComponentReflection { docstring_md: "How we interpret the coordinate system of an entity/space.\n\nFor instance: What is \"up\"? What does the Z axis mean?\n\nThe three coordinates are always ordered as [x, y, z].\n\nFor example [Right, Down, Forward] means that the X axis points to the right, the Y axis points\ndown, and the Z axis points forward.\n\n⚠ [Rerun does not yet support left-handed coordinate systems](https://github.com/rerun-io/rerun/issues/5032).\n\nThe following constants are used to represent the different directions:\n * Up = 1\n * Down = 2\n * Right = 3\n * Left = 4\n * Forward = 5\n * Back = 6", - custom_placeholder: Some(ViewCoordinates::default().to_arrow2()?), + custom_placeholder: Some(ViewCoordinates::default().to_arrow()?), datatype: ViewCoordinates::arrow2_datatype(), }, ), diff --git a/crates/store/re_types_core/src/loggable_batch.rs b/crates/store/re_types_core/src/loggable_batch.rs index 665545a1ca6e..71e86b4f7692 100644 --- a/crates/store/re_types_core/src/loggable_batch.rs +++ b/crates/store/re_types_core/src/loggable_batch.rs @@ -24,6 +24,11 @@ pub trait LoggableBatch { // type Loggable: Loggable; /// Serializes the batch into an Arrow array. + fn to_arrow(&self) -> SerializationResult { + self.to_arrow2().map(|array| array.into()) + } + + /// Serializes the batch into an Arrow2 array. fn to_arrow2(&self) -> SerializationResult>; } diff --git a/crates/store/re_types_core/src/reflection.rs b/crates/store/re_types_core/src/reflection.rs index 9fc71d5b4fdf..7ff274c54a3a 100644 --- a/crates/store/re_types_core/src/reflection.rs +++ b/crates/store/re_types_core/src/reflection.rs @@ -1,5 +1,7 @@ //! Run-time reflection for reading meta-data about components and archetypes. +use arrow::array::ArrayRef; + use crate::{ArchetypeName, ComponentName}; /// A trait for code-generated enums. @@ -218,7 +220,7 @@ pub struct ComponentReflection { /// especially when it's necessary to have a starting value for edit ui. /// Typically, this is only used when `FallbackProvider`s are not available. /// If there's no custom placeholder, a placeholder can be derived from the arrow datatype. - pub custom_placeholder: Option>, + pub custom_placeholder: Option, /// Datatype of the component. pub datatype: arrow2::datatypes::DataType, diff --git a/crates/viewer/re_data_ui/src/component.rs b/crates/viewer/re_data_ui/src/component.rs index 663923ddb5cd..c49af8d75113 100644 --- a/crates/viewer/re_data_ui/src/component.rs +++ b/crates/viewer/re_data_ui/src/component.rs @@ -33,7 +33,7 @@ impl DataUi for ComponentPathLatestAtResults<'_> { let Some(num_instances) = self .unit - .component_batch_raw(component_name) + .component_batch_raw_arrow2(component_name) .map(|data| data.len()) else { ui.weak(""); diff --git a/crates/viewer/re_selection_panel/src/defaults_ui.rs b/crates/viewer/re_selection_panel/src/defaults_ui.rs index 1800b68ca702..adcbadb7b77a 100644 --- a/crates/viewer/re_selection_panel/src/defaults_ui.rs +++ b/crates/viewer/re_selection_panel/src/defaults_ui.rs @@ -304,7 +304,10 @@ fn add_popup_ui( .with_row( RowId::new(), ctx.blueprint_timepoint_for_writes(), - [(ComponentDescriptor::new(component_name), initial_data)], + [( + ComponentDescriptor::new(component_name), + initial_data.into(), + )], ) .build() { diff --git a/crates/viewer/re_selection_panel/src/visualizer_ui.rs b/crates/viewer/re_selection_panel/src/visualizer_ui.rs index 9f10aac8862a..e8e04c99ad15 100644 --- a/crates/viewer/re_selection_panel/src/visualizer_ui.rs +++ b/crates/viewer/re_selection_panel/src/visualizer_ui.rs @@ -162,7 +162,7 @@ fn visualizer_components( component_name: &ComponentName, ) -> Option<(Option, Box)> { let unit = unit?; - let batch = unit.component_batch_raw(component_name)?; + let batch = unit.component_batch_raw_arrow2(component_name)?; if batch.is_empty() { None } else { @@ -205,9 +205,10 @@ fn visualizer_components( let result_default = query_result.defaults.get(&component_name); let raw_default = non_empty_component_batch_raw(result_default, &component_name); - let raw_fallback = visualizer + let raw_fallback: Box = visualizer .fallback_provider() - .fallback_for(&query_ctx, component_name); + .fallback_for(&query_ctx, component_name) + .into(); // Determine where the final value comes from. // Putting this into an enum makes it easier to reason about the next steps. @@ -485,7 +486,7 @@ fn menu_more( let override_differs_from_default = raw_override != &ctx .viewer_ctx - .raw_latest_at_in_default_blueprint(override_path, component_name); + .raw_latest_at_in_default_blueprint_arrow2(override_path, component_name); if ui .add_enabled( override_differs_from_default, diff --git a/crates/viewer/re_view/Cargo.toml b/crates/viewer/re_view/Cargo.toml index 6bfd5705e581..29e1973862a3 100644 --- a/crates/viewer/re_view/Cargo.toml +++ b/crates/viewer/re_view/Cargo.toml @@ -37,6 +37,7 @@ re_viewport_blueprint.workspace = true ahash.workspace = true arrow.workspace = true +arrow2.workspace = true bytemuck.workspace = true egui.workspace = true glam.workspace = true diff --git a/crates/viewer/re_view/src/query.rs b/crates/viewer/re_view/src/query.rs index 173a6e453831..7365c2324cc8 100644 --- a/crates/viewer/re_view/src/query.rs +++ b/crates/viewer/re_view/src/query.rs @@ -1,10 +1,11 @@ +use arrow::array::ArrayRef; use nohash_hasher::IntSet; use crate::{ results_ext::{HybridLatestAtResults, HybridRangeResults}, HybridResults, }; -use re_chunk_store::{external::re_chunk::Arrow2Array, LatestAtQuery, RangeQuery, RowId}; +use re_chunk_store::{LatestAtQuery, RangeQuery, RowId}; use re_log_types::{TimeInt, Timeline}; use re_query::LatestAtResults; use re_types_core::{Archetype, ComponentName}; @@ -225,7 +226,7 @@ pub trait DataResultQuery { query_ctx: &'a QueryContext<'a>, visualizer_collection: &'a re_viewer_context::VisualizerCollection, component: re_types_core::ComponentName, - ) -> Box; + ) -> ArrayRef; } impl DataResultQuery for DataResult { @@ -265,7 +266,7 @@ impl DataResultQuery for DataResult { query_ctx: &'a QueryContext<'a>, visualizer_collection: &'a re_viewer_context::VisualizerCollection, component: re_types_core::ComponentName, - ) -> Box { + ) -> ArrayRef { // TODO(jleibs): This should be cached somewhere for vis in &self.visualizers { let Ok(vis) = visualizer_collection.get_by_identifier(*vis) else { diff --git a/crates/viewer/re_view/src/results_ext.rs b/crates/viewer/re_view/src/results_ext.rs index e69e764482f0..60c4e126a67b 100644 --- a/crates/viewer/re_view/src/results_ext.rs +++ b/crates/viewer/re_view/src/results_ext.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use itertools::Itertools as _; use re_chunk_store::{Chunk, LatestAtQuery, RangeQuery, UnitChunkShared}; -use re_log_types::external::arrow2::array::Array as Arrow2Array; use re_log_types::hash::Hash64; use re_query::{LatestAtResults, RangeResults}; use re_types_core::ComponentName; @@ -50,7 +49,7 @@ impl HybridLatestAtResults<'_> { .or_else(|| self.defaults.get(&component_name)) } - pub fn fallback_raw(&self, component_name: ComponentName) -> Box { + pub fn fallback_raw(&self, component_name: ComponentName) -> arrow::array::ArrayRef { let query_context = QueryContext { viewer_ctx: self.ctx.viewer_ctx, target_entity_path: &self.data_result.entity_path, @@ -118,7 +117,7 @@ impl HybridLatestAtResults<'_> { .or_else(|| { // No override, no store, no default -> try fallback instead let raw_fallback = self.fallback_raw(C::name()); - C::from_arrow2(raw_fallback.as_ref()) + C::from_arrow(raw_fallback.as_ref()) .ok() .and_then(|r| r.first().cloned()) }) diff --git a/crates/viewer/re_view/src/view_property_ui.rs b/crates/viewer/re_view/src/view_property_ui.rs index 995a3825803b..1307c2d41ea4 100644 --- a/crates/viewer/re_view/src/view_property_ui.rs +++ b/crates/viewer/re_view/src/view_property_ui.rs @@ -89,7 +89,9 @@ pub fn view_property_component_ui( field: &ArchetypeFieldReflection, fallback_provider: &dyn ComponentFallbackProvider, ) { - let component_array = property.component_raw(field.component_name); + let component_array: Option> = property + .component_raw(field.component_name) + .map(|array| array.into()); let row_id = property.component_row_id(field.component_name); let ui_types = ctx diff --git a/crates/viewer/re_view_graph/src/ui/selection.rs b/crates/viewer/re_view_graph/src/ui/selection.rs index d8b8906051bc..659e4dbb120e 100644 --- a/crates/viewer/re_view_graph/src/ui/selection.rs +++ b/crates/viewer/re_view_graph/src/ui/selection.rs @@ -1,3 +1,4 @@ +use re_chunk::Arrow2Array; use re_types::{ blueprint::components::Enabled, Archetype, ArchetypeReflectionMarker, Component as _, }; @@ -61,7 +62,9 @@ pub fn view_property_force_ui( .find(|field| field.component_name == Enabled::name()) .expect("forces are required to have an `Enabled` component"); - let component_array = property.component_raw(field.component_name); + let component_array: Option> = property + .component_raw(field.component_name) + .map(|array| array.into()); let row_id = property.component_row_id(field.component_name); let singleline_ui: &dyn Fn(&mut egui::Ui) = &|ui| { diff --git a/crates/viewer/re_viewer_context/Cargo.toml b/crates/viewer/re_viewer_context/Cargo.toml index a0768622da06..a541a6600d27 100644 --- a/crates/viewer/re_viewer_context/Cargo.toml +++ b/crates/viewer/re_viewer_context/Cargo.toml @@ -40,6 +40,7 @@ re_video = { workspace = true, features = ["serde"] } ahash.workspace = true anyhow.workspace = true +arrow.workspace = true bit-vec.workspace = true bitflags.workspace = true bytemuck.workspace = true diff --git a/crates/viewer/re_viewer_context/src/blueprint_helpers.rs b/crates/viewer/re_viewer_context/src/blueprint_helpers.rs index ac0265998a38..c3912a3da785 100644 --- a/crates/viewer/re_viewer_context/src/blueprint_helpers.rs +++ b/crates/viewer/re_viewer_context/src/blueprint_helpers.rs @@ -1,3 +1,4 @@ +use arrow::array::ArrayRef; use re_chunk::{Arrow2Array, RowId}; use re_chunk_store::external::re_chunk::Chunk; use re_log_types::{EntityPath, TimeInt, TimePoint, Timeline}; @@ -126,7 +127,7 @@ impl ViewerContext<'_> { &self, entity_path: &EntityPath, component_name: ComponentName, - ) -> Option> { + ) -> Option { self.store_context .default_blueprint .and_then(|default_blueprint| { @@ -137,6 +138,16 @@ impl ViewerContext<'_> { }) } + /// Queries a raw component from the default blueprint. + pub fn raw_latest_at_in_default_blueprint_arrow2( + &self, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> Option> { + self.raw_latest_at_in_default_blueprint(entity_path, component_name) + .map(|array| array.into()) + } + /// Resets a blueprint component to the value it had in the default blueprint. pub fn reset_blueprint_component_by_name( &self, @@ -144,7 +155,7 @@ impl ViewerContext<'_> { component_name: ComponentName, ) { if let Some(default_value) = - self.raw_latest_at_in_default_blueprint(entity_path, component_name) + self.raw_latest_at_in_default_blueprint_arrow2(entity_path, component_name) { self.save_blueprint_array(entity_path, component_name, default_value); } else { @@ -164,7 +175,7 @@ impl ViewerContext<'_> { .latest_at(self.blueprint_query, entity_path, [component_name]) .get(&component_name) .and_then(|unit| { - unit.component_batch_raw(&component_name) + unit.component_batch_raw_arrow2(&component_name) .map(|array| array.data_type().clone()) }) else { diff --git a/crates/viewer/re_viewer_context/src/component_fallbacks.rs b/crates/viewer/re_viewer_context/src/component_fallbacks.rs index fba8527b51f0..0ad9e0babdb6 100644 --- a/crates/viewer/re_viewer_context/src/component_fallbacks.rs +++ b/crates/viewer/re_viewer_context/src/component_fallbacks.rs @@ -1,11 +1,13 @@ -use re_types::{external::arrow2, ComponentName}; +use arrow::array::ArrayRef; + +use re_types::ComponentName; use crate::QueryContext; /// Result for a fallback request to a provider. pub enum ComponentFallbackProviderResult { /// A fallback value was successfully provided. - Value(Box), + Value(ArrayRef), /// The fallback provider is not able to handle the given component. /// @@ -20,7 +22,7 @@ pub enum ComponentFallbackProviderResult { impl From for ComponentFallbackProviderResult { fn from(batch: T) -> Self { - match batch.to_arrow2() { + match batch.to_arrow() { Ok(value) => Self::Value(value), Err(err) => Self::SerializationError(err), } @@ -53,11 +55,7 @@ pub trait ComponentFallbackProvider { /// Provides a fallback value for a given component, first trying the provider and /// then falling back to the placeholder value registered in the viewer context. - fn fallback_for( - &self, - ctx: &QueryContext<'_>, - component: ComponentName, - ) -> Box { + fn fallback_for(&self, ctx: &QueryContext<'_>, component: ComponentName) -> ArrayRef { match self.try_provide_fallback(ctx, component) { ComponentFallbackProviderResult::Value(value) => { return value; diff --git a/crates/viewer/re_viewer_context/src/component_ui_registry.rs b/crates/viewer/re_viewer_context/src/component_ui_registry.rs index 3507aaa423cc..1e93023de52e 100644 --- a/crates/viewer/re_viewer_context/src/component_ui_registry.rs +++ b/crates/viewer/re_viewer_context/src/component_ui_registry.rs @@ -391,7 +391,7 @@ impl ComponentUiRegistry { // Don't use component.raw_instance here since we want to handle the case where there's several // elements differently. // Also, it allows us to slice the array without cloning any elements. - let Some(array) = unit.component_batch_raw(&component_name) else { + let Some(array) = unit.component_batch_raw_arrow2(&component_name) else { re_log::error_once!("Couldn't get {component_name}: missing"); ui.error_with_details_on_hover(format!("Couldn't get {component_name}: missing")); return; @@ -573,7 +573,7 @@ impl ComponentUiRegistry { { component_raw } else { - fallback_provider.fallback_for(ctx, component_name) + fallback_provider.fallback_for(ctx, component_name).into() }; self.edit_ui_raw( diff --git a/crates/viewer/re_viewer_context/src/viewer_context.rs b/crates/viewer/re_viewer_context/src/viewer_context.rs index c4210930819d..e5720a6404d7 100644 --- a/crates/viewer/re_viewer_context/src/viewer_context.rs +++ b/crates/viewer/re_viewer_context/src/viewer_context.rs @@ -1,4 +1,5 @@ use ahash::HashMap; +use arrow::array::ArrayRef; use parking_lot::RwLock; use re_chunk_store::LatestAtQuery; @@ -227,10 +228,7 @@ impl ViewerContext<'_> { /// The rationale is that to get into this situation, we need to know of a component name for which /// we don't have a datatype, meaning that we can't make any statement about what data this component should represent. // TODO(andreas): Are there cases where this is expected and how to handle this? - pub fn placeholder_for( - &self, - component: re_chunk::ComponentName, - ) -> Box { + pub fn placeholder_for(&self, component: re_chunk::ComponentName) -> ArrayRef { let datatype = if let Some(reflection) = self.reflection.components.get(&component) { // It's a builtin type with reflection. We either have custom place holder, or can rely on the known datatype. if let Some(placeholder) = reflection.custom_placeholder.as_ref() { @@ -251,7 +249,7 @@ impl ViewerContext<'_> { // TODO(andreas): Is this operation common enough to cache the result? If so, here or in the reflection data? // The nice thing about this would be that we could always give out references (but updating said cache wouldn't be easy in that case). - re_types::reflection::generic_placeholder_for_datatype(&datatype) + re_types::reflection::generic_placeholder_for_datatype(&datatype).into() } } diff --git a/crates/viewer/re_viewport_blueprint/Cargo.toml b/crates/viewer/re_viewport_blueprint/Cargo.toml index e4a982608ae9..de84226d4558 100644 --- a/crates/viewer/re_viewport_blueprint/Cargo.toml +++ b/crates/viewer/re_viewport_blueprint/Cargo.toml @@ -31,6 +31,7 @@ re_ui.workspace = true re_viewer_context.workspace = true ahash.workspace = true +arrow.workspace = true egui_tiles.workspace = true egui.workspace = true itertools.workspace = true diff --git a/crates/viewer/re_viewport_blueprint/src/view_properties.rs b/crates/viewer/re_viewport_blueprint/src/view_properties.rs index 4df9945f0a93..a63b923c7b67 100644 --- a/crates/viewer/re_viewport_blueprint/src/view_properties.rs +++ b/crates/viewer/re_viewport_blueprint/src/view_properties.rs @@ -1,9 +1,7 @@ use re_chunk_store::LatestAtQuery; use re_entity_db::{external::re_query::LatestAtResults, EntityDb}; use re_log_types::EntityPath; -use re_types::{ - external::arrow2, Archetype, ArchetypeName, ComponentBatch, ComponentName, DeserializationError, -}; +use re_types::{Archetype, ArchetypeName, ComponentBatch, ComponentName, DeserializationError}; use re_viewer_context::{ external::re_entity_db::EntityTree, ComponentFallbackError, ComponentFallbackProvider, QueryContext, ViewId, ViewSystemExecutionError, ViewerContext, @@ -113,7 +111,7 @@ impl ViewProperty { view_state: &dyn re_viewer_context::ViewState, ) -> Result, ViewPropertyQueryError> { let component_name = C::name(); - C::from_arrow2( + C::from_arrow( self.component_or_fallback_raw(ctx, component_name, fallback_provider, view_state) .as_ref(), ) @@ -135,7 +133,7 @@ impl ViewProperty { ) -> Result>, DeserializationError> { let component_name = C::name(); self.component_raw(component_name) - .map(|raw| C::from_arrow2(raw.as_ref())) + .map(|raw| C::from_arrow(raw.as_ref())) .transpose() } @@ -153,13 +151,11 @@ impl ViewProperty { .and_then(|unit| unit.row_id()) } - pub fn component_raw( - &self, - component_name: ComponentName, - ) -> Option> { - self.query_results - .get(&component_name) - .and_then(|unit| unit.component_batch_raw(&component_name)) + pub fn component_raw(&self, component_name: ComponentName) -> Option { + self.query_results.get(&component_name).and_then(|unit| { + unit.component_batch_raw_arrow2(&component_name) + .map(|array| array.into()) + }) } fn component_or_fallback_raw( @@ -168,7 +164,7 @@ impl ViewProperty { component_name: ComponentName, fallback_provider: &dyn ComponentFallbackProvider, view_state: &dyn re_viewer_context::ViewState, - ) -> Box { + ) -> arrow::array::ArrayRef { if let Some(value) = self.component_raw(component_name) { if value.len() > 0 { return value; From f3e0608d395ff2d06359da3b47814cb1f3cbe78e Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 17:52:24 +0100 Subject: [PATCH 2/9] Break out arrow2 size estimation to its own file --- .../arrow2_sizes.rs} | 502 +++++------------- .../store/re_types_core/src/size_bytes/mod.rs | 269 ++++++++++ 2 files changed, 388 insertions(+), 383 deletions(-) rename crates/store/re_types_core/src/{size_bytes.rs => size_bytes/arrow2_sizes.rs} (71%) create mode 100644 crates/store/re_types_core/src/size_bytes/mod.rs diff --git a/crates/store/re_types_core/src/size_bytes.rs b/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs similarity index 71% rename from crates/store/re_types_core/src/size_bytes.rs rename to crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs index 3c821883eb08..6039c4bdcc4a 100644 --- a/crates/store/re_types_core/src/size_bytes.rs +++ b/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs @@ -1,386 +1,23 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; -use std::sync::Arc; - -use arrow2::datatypes::{DataType, Field}; -use arrow2::types::{NativeType, Offset}; -use smallvec::SmallVec; - -// --- - -/// Approximations of stack and heap size for both internal and external types. -/// -/// Motly used for statistics and triggering events such as garbage collection. -pub trait SizeBytes { - /// Returns the total size of `self` in bytes, accounting for both stack and heap space. - #[inline] - fn total_size_bytes(&self) -> u64 { - self.stack_size_bytes() + self.heap_size_bytes() - } - - /// Returns the total size of `self` on the stack, in bytes. - /// - /// Defaults to `std::mem::size_of_val(self)`. - #[inline] - fn stack_size_bytes(&self) -> u64 { - std::mem::size_of_val(self) as _ - } - - /// Returns the total size of `self` on the heap, in bytes. - fn heap_size_bytes(&self) -> u64; - - /// Is `Self` just plain old data? - /// - /// If `true`, this will make most blanket implementations of `SizeBytes` much faster (e.g. `Vec`). - #[inline] - fn is_pod() -> bool { - false - } -} - -// TODO(rust-lang/rust#31844): This isn't happening without specialization. -// impl SizeBytes for T where T: bytemuck::Pod { … } - -// --- Std --- - -impl SizeBytes for String { - /// Does not take capacity into account. - #[inline] - fn heap_size_bytes(&self) -> u64 { - self.as_bytes().len() as u64 - } -} - -impl SizeBytes for BTreeMap { - #[inline] - fn heap_size_bytes(&self) -> u64 { - // NOTE: It's all on the heap at this point. - - let keys_size_bytes = if K::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.keys().map(SizeBytes::total_size_bytes).sum::() - }; - - let values_size_bytes = if V::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.values().map(SizeBytes::total_size_bytes).sum::() - }; - - keys_size_bytes + values_size_bytes - } -} - -impl SizeBytes for BTreeSet { - #[inline] - fn heap_size_bytes(&self) -> u64 { - // NOTE: It's all on the heap at this point. - - if K::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.iter().map(SizeBytes::total_size_bytes).sum::() - } - } -} - -impl SizeBytes for HashMap { - #[inline] - fn heap_size_bytes(&self) -> u64 { - // NOTE: It's all on the heap at this point. - - let keys_size_bytes = if K::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.keys().map(SizeBytes::total_size_bytes).sum::() - }; - - let values_size_bytes = if V::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.values().map(SizeBytes::total_size_bytes).sum::() - }; - - keys_size_bytes + values_size_bytes - } -} - -// NOTE: Do _not_ implement `SizeBytes` for slices: we cannot know whether they point to the stack -// or the heap! - -impl SizeBytes for [T; N] { - #[inline] - fn heap_size_bytes(&self) -> u64 { - if T::is_pod() { - 0 // it's a const-sized array - } else { - self.iter().map(SizeBytes::heap_size_bytes).sum::() - } - } -} - -impl SizeBytes for Vec { - /// Does not take capacity into account. - #[inline] - fn heap_size_bytes(&self) -> u64 { - // NOTE: It's all on the heap at this point. - if T::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.iter().map(SizeBytes::total_size_bytes).sum::() - } - } -} - -impl SizeBytes for VecDeque { - /// Does not take capacity into account. - #[inline] - fn heap_size_bytes(&self) -> u64 { - // NOTE: It's all on the heap at this point. - if T::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.iter().map(SizeBytes::total_size_bytes).sum::() - } - } -} - -impl SizeBytes for SmallVec<[T; N]> { - /// Does not take capacity into account. - #[inline] - fn heap_size_bytes(&self) -> u64 { - if self.len() <= N { - // The `SmallVec` is still smaller than the threshold so no heap data has been - // allocated yet, beyond the heap data each element might have. - - if T::is_pod() { - 0 // early-out - } else { - self.iter().map(SizeBytes::heap_size_bytes).sum::() - } - } else { - // NOTE: It's all on the heap at this point. - if T::is_pod() { - (self.len() * std::mem::size_of::()) as _ - } else { - self.iter().map(SizeBytes::total_size_bytes).sum::() - } - } - } -} - -impl SizeBytes for Option { - #[inline] - fn heap_size_bytes(&self) -> u64 { - self.as_ref().map_or(0, SizeBytes::heap_size_bytes) - } -} - -impl SizeBytes for Arc { - #[inline] - fn heap_size_bytes(&self) -> u64 { - 0 // assume it's amortized - } -} - -impl SizeBytes for Box { - #[inline] - fn heap_size_bytes(&self) -> u64 { - T::total_size_bytes(&**self) - } -} - -// TODO(rust-lang/rust#31844): `impl SizeBytesExt for T {}` would be nice but -// violates orphan rules. -macro_rules! impl_size_bytes_pod { - ($ty:ty) => { - impl SizeBytes for $ty { - #[inline] - fn heap_size_bytes(&self) -> u64 { - 0 - } - - #[inline] - fn is_pod() -> bool { - true - } - } - }; - ($ty:ty, $($rest:ty),+) => { - impl_size_bytes_pod!($ty); impl_size_bytes_pod!($($rest),+); - }; -} - -impl_size_bytes_pod!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, bool, f32, f64); -impl_size_bytes_pod!(half::f16); - -impl SizeBytes for (T, U) -where - T: SizeBytes, - U: SizeBytes, -{ - #[inline] - fn heap_size_bytes(&self) -> u64 { - let (a, b) = self; - a.heap_size_bytes() + b.heap_size_bytes() - } - - #[inline] - fn is_pod() -> bool { - T::is_pod() && U::is_pod() - } -} - -impl SizeBytes for (T, U, V) -where - T: SizeBytes, - U: SizeBytes, - V: SizeBytes, -{ - #[inline] - fn heap_size_bytes(&self) -> u64 { - let (a, b, c) = self; - a.heap_size_bytes() + b.heap_size_bytes() + c.heap_size_bytes() - } - - #[inline] - fn is_pod() -> bool { - T::is_pod() && U::is_pod() && V::is_pod() - } -} - -impl SizeBytes for (T, U, V, W) -where - T: SizeBytes, - U: SizeBytes, - V: SizeBytes, - W: SizeBytes, -{ - #[inline] - fn heap_size_bytes(&self) -> u64 { - let (a, b, c, d) = self; - a.heap_size_bytes() + b.heap_size_bytes() + c.heap_size_bytes() + d.heap_size_bytes() - } - - #[inline] - fn is_pod() -> bool { - T::is_pod() && U::is_pod() && V::is_pod() && W::is_pod() - } -} - -// --- Arrow --- - -impl SizeBytes for DataType { - #[inline] - fn heap_size_bytes(&self) -> u64 { - match self { - Self::Null - | Self::Binary - | Self::Boolean - | Self::Date32 - | Self::Date64 - | Self::Float16 - | Self::Float32 - | Self::Float64 - | Self::Int16 - | Self::Int32 - | Self::Int64 - | Self::Int8 - | Self::LargeBinary - | Self::LargeUtf8 - | Self::UInt16 - | Self::UInt32 - | Self::UInt64 - | Self::UInt8 - | Self::Time32(_) - | Self::Time64(_) - | Self::Duration(_) - | Self::Interval(_) - | Self::FixedSizeBinary(_) - | Self::Decimal(_, _) - | Self::Decimal256(_, _) - | Self::Utf8 => 0, - Self::Timestamp(_, str) => str.heap_size_bytes(), - Self::List(field) - | Self::FixedSizeList(field, _) - | Self::LargeList(field) - | Self::Map(field, _) => field.total_size_bytes(), // NOTE: Boxed, it's all on the heap - Self::Struct(fields) => fields.heap_size_bytes(), - Self::Union(fields, indices, _) => fields.heap_size_bytes() + indices.heap_size_bytes(), - Self::Dictionary(_, datatype, _) => datatype.total_size_bytes(), // NOTE: Boxed, it's all on the heap - Self::Extension(name, datatype, extra) => { - name.heap_size_bytes() - + datatype.total_size_bytes() // NOTE: Boxed, it's all on the heap - + extra.heap_size_bytes() - } - } - } -} - -impl SizeBytes for Field { - #[inline] - fn heap_size_bytes(&self) -> u64 { - let Self { - name, - data_type, - is_nullable, - metadata, - } = self; - - name.heap_size_bytes() - + data_type.heap_size_bytes() - + is_nullable.heap_size_bytes() - + metadata.heap_size_bytes() - } -} - -impl SizeBytes for dyn Array { - #[inline] - fn heap_size_bytes(&self) -> u64 { - estimated_bytes_size(self) as _ - } -} - -impl SizeBytes for Box { - #[inline] - fn heap_size_bytes(&self) -> u64 { - estimated_bytes_size(&**self as _) as _ - } -} - -impl SizeBytes for PrimitiveArray { - #[inline] - fn heap_size_bytes(&self) -> u64 { - estimated_bytes_size(self) as _ - } -} - -impl SizeBytes for ListArray { - #[inline] - fn heap_size_bytes(&self) -> u64 { - estimated_bytes_size(self) as _ - } -} - -impl SizeBytes for StructArray { - #[inline] - fn heap_size_bytes(&self) -> u64 { - estimated_bytes_size(self) as _ - } -} - -// --- Arrow estimations --- - -// The following is a modified version of [1], available under MIT OR Apache-2.0. -// -// [1] https://github.com/jorgecarleitao/arrow2/blob/v0.16.0/src/compute/aggregate/memory.rs - -use arrow2::array::{ - Array, BinaryArray, BooleanArray, DictionaryArray, FixedSizeBinaryArray, FixedSizeListArray, - ListArray, MapArray, PrimitiveArray, StructArray, UnionArray, Utf8Array, +//! --- Arrow2 size estimations --- +//! +//! The following is a modified version of [1], available under MIT OR Apache-2.0. +//! +//! [1] + +use std::collections::BTreeMap; + +use arrow2::{ + array::{ + Array, BinaryArray, BooleanArray, DictionaryArray, FixedSizeBinaryArray, + FixedSizeListArray, ListArray, MapArray, PrimitiveArray, StructArray, UnionArray, + Utf8Array, + }, + bitmap::Bitmap, + datatypes::{DataType, Field, PhysicalType}, + types::{NativeType, Offset}, }; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::PhysicalType; + +use super::SizeBytes; macro_rules! with_match_primitive_type {( $key_type:expr, | $_:tt $T:ident | $($body:tt)* @@ -630,7 +267,7 @@ fn estimated_bytes_size(array: &dyn Array) -> usize { // Arrow array?". #[test] #[allow(clippy::from_iter_instead_of_collect)] -fn test_arrow_estimated_size_bytes() { +fn test_arrow2_estimated_size_bytes() { use arrow2::{ array::{Array, Float64Array, ListArray, StructArray, UInt64Array, Utf8Array}, buffer::Buffer, @@ -814,3 +451,102 @@ fn test_arrow_estimated_size_bytes() { assert_eq!(raw_size_bytes, arrow_size_bytes); } } + +impl SizeBytes for DataType { + #[inline] + fn heap_size_bytes(&self) -> u64 { + match self { + Self::Null + | Self::Binary + | Self::Boolean + | Self::Date32 + | Self::Date64 + | Self::Float16 + | Self::Float32 + | Self::Float64 + | Self::Int16 + | Self::Int32 + | Self::Int64 + | Self::Int8 + | Self::LargeBinary + | Self::LargeUtf8 + | Self::UInt16 + | Self::UInt32 + | Self::UInt64 + | Self::UInt8 + | Self::Time32(_) + | Self::Time64(_) + | Self::Duration(_) + | Self::Interval(_) + | Self::FixedSizeBinary(_) + | Self::Decimal(_, _) + | Self::Decimal256(_, _) + | Self::Utf8 => 0, + Self::Timestamp(_, str) => str.heap_size_bytes(), + Self::List(field) + | Self::FixedSizeList(field, _) + | Self::LargeList(field) + | Self::Map(field, _) => field.total_size_bytes(), // NOTE: Boxed, it's all on the heap + Self::Struct(fields) => fields.heap_size_bytes(), + Self::Union(fields, indices, _) => fields.heap_size_bytes() + indices.heap_size_bytes(), + Self::Dictionary(_, datatype, _) => datatype.total_size_bytes(), // NOTE: Boxed, it's all on the heap + Self::Extension(name, datatype, extra) => { + name.heap_size_bytes() + + datatype.total_size_bytes() // NOTE: Boxed, it's all on the heap + + extra.heap_size_bytes() + } + } + } +} + +impl SizeBytes for Field { + #[inline] + fn heap_size_bytes(&self) -> u64 { + let Self { + name, + data_type, + is_nullable, + metadata, + } = self; + + name.heap_size_bytes() + + data_type.heap_size_bytes() + + is_nullable.heap_size_bytes() + + metadata.heap_size_bytes() + } +} + +impl SizeBytes for dyn Array { + #[inline] + fn heap_size_bytes(&self) -> u64 { + estimated_bytes_size(self) as _ + } +} + +impl SizeBytes for Box { + #[inline] + fn heap_size_bytes(&self) -> u64 { + estimated_bytes_size(&**self as _) as _ + } +} + +impl SizeBytes for PrimitiveArray { + #[inline] + fn heap_size_bytes(&self) -> u64 { + estimated_bytes_size(self) as _ + } +} + +impl SizeBytes for ListArray { + #[inline] + fn heap_size_bytes(&self) -> u64 { + estimated_bytes_size(self) as _ + } +} + +impl SizeBytes for StructArray { + #[inline] + fn heap_size_bytes(&self) -> u64 { + estimated_bytes_size(self) as _ + } +} diff --git a/crates/store/re_types_core/src/size_bytes/mod.rs b/crates/store/re_types_core/src/size_bytes/mod.rs new file mode 100644 index 000000000000..c05a1bf1f5c4 --- /dev/null +++ b/crates/store/re_types_core/src/size_bytes/mod.rs @@ -0,0 +1,269 @@ +mod arrow2_sizes; + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::sync::Arc; + +use smallvec::SmallVec; + +// --- + +/// Approximations of stack and heap size for both internal and external types. +/// +/// Motly used for statistics and triggering events such as garbage collection. +pub trait SizeBytes { + /// Returns the total size of `self` in bytes, accounting for both stack and heap space. + #[inline] + fn total_size_bytes(&self) -> u64 { + self.stack_size_bytes() + self.heap_size_bytes() + } + + /// Returns the total size of `self` on the stack, in bytes. + /// + /// Defaults to `std::mem::size_of_val(self)`. + #[inline] + fn stack_size_bytes(&self) -> u64 { + std::mem::size_of_val(self) as _ + } + + /// Returns the total size of `self` on the heap, in bytes. + fn heap_size_bytes(&self) -> u64; + + /// Is `Self` just plain old data? + /// + /// If `true`, this will make most blanket implementations of `SizeBytes` much faster (e.g. `Vec`). + #[inline] + fn is_pod() -> bool { + false + } +} + +// TODO(rust-lang/rust#31844): This isn't happening without specialization. +// impl SizeBytes for T where T: bytemuck::Pod { … } + +// --- Std --- + +impl SizeBytes for String { + /// Does not take capacity into account. + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.as_bytes().len() as u64 + } +} + +impl SizeBytes for BTreeMap { + #[inline] + fn heap_size_bytes(&self) -> u64 { + // NOTE: It's all on the heap at this point. + + let keys_size_bytes = if K::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.keys().map(SizeBytes::total_size_bytes).sum::() + }; + + let values_size_bytes = if V::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.values().map(SizeBytes::total_size_bytes).sum::() + }; + + keys_size_bytes + values_size_bytes + } +} + +impl SizeBytes for BTreeSet { + #[inline] + fn heap_size_bytes(&self) -> u64 { + // NOTE: It's all on the heap at this point. + + if K::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.iter().map(SizeBytes::total_size_bytes).sum::() + } + } +} + +impl SizeBytes for HashMap { + #[inline] + fn heap_size_bytes(&self) -> u64 { + // NOTE: It's all on the heap at this point. + + let keys_size_bytes = if K::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.keys().map(SizeBytes::total_size_bytes).sum::() + }; + + let values_size_bytes = if V::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.values().map(SizeBytes::total_size_bytes).sum::() + }; + + keys_size_bytes + values_size_bytes + } +} + +// NOTE: Do _not_ implement `SizeBytes` for slices: we cannot know whether they point to the stack +// or the heap! + +impl SizeBytes for [T; N] { + #[inline] + fn heap_size_bytes(&self) -> u64 { + if T::is_pod() { + 0 // it's a const-sized array + } else { + self.iter().map(SizeBytes::heap_size_bytes).sum::() + } + } +} + +impl SizeBytes for Vec { + /// Does not take capacity into account. + #[inline] + fn heap_size_bytes(&self) -> u64 { + // NOTE: It's all on the heap at this point. + if T::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.iter().map(SizeBytes::total_size_bytes).sum::() + } + } +} + +impl SizeBytes for VecDeque { + /// Does not take capacity into account. + #[inline] + fn heap_size_bytes(&self) -> u64 { + // NOTE: It's all on the heap at this point. + if T::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.iter().map(SizeBytes::total_size_bytes).sum::() + } + } +} + +impl SizeBytes for SmallVec<[T; N]> { + /// Does not take capacity into account. + #[inline] + fn heap_size_bytes(&self) -> u64 { + if self.len() <= N { + // The `SmallVec` is still smaller than the threshold so no heap data has been + // allocated yet, beyond the heap data each element might have. + + if T::is_pod() { + 0 // early-out + } else { + self.iter().map(SizeBytes::heap_size_bytes).sum::() + } + } else { + // NOTE: It's all on the heap at this point. + if T::is_pod() { + (self.len() * std::mem::size_of::()) as _ + } else { + self.iter().map(SizeBytes::total_size_bytes).sum::() + } + } + } +} + +impl SizeBytes for Option { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.as_ref().map_or(0, SizeBytes::heap_size_bytes) + } +} + +impl SizeBytes for Arc { + #[inline] + fn heap_size_bytes(&self) -> u64 { + 0 // assume it's amortized + } +} + +impl SizeBytes for Box { + #[inline] + fn heap_size_bytes(&self) -> u64 { + T::total_size_bytes(&**self) + } +} + +// TODO(rust-lang/rust#31844): `impl SizeBytesExt for T {}` would be nice but +// violates orphan rules. +macro_rules! impl_size_bytes_pod { + ($ty:ty) => { + impl SizeBytes for $ty { + #[inline] + fn heap_size_bytes(&self) -> u64 { + 0 + } + + #[inline] + fn is_pod() -> bool { + true + } + } + }; + ($ty:ty, $($rest:ty),+) => { + impl_size_bytes_pod!($ty); impl_size_bytes_pod!($($rest),+); + }; +} + +impl_size_bytes_pod!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, bool, f32, f64); +impl_size_bytes_pod!(half::f16); + +impl SizeBytes for (T, U) +where + T: SizeBytes, + U: SizeBytes, +{ + #[inline] + fn heap_size_bytes(&self) -> u64 { + let (a, b) = self; + a.heap_size_bytes() + b.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + T::is_pod() && U::is_pod() + } +} + +impl SizeBytes for (T, U, V) +where + T: SizeBytes, + U: SizeBytes, + V: SizeBytes, +{ + #[inline] + fn heap_size_bytes(&self) -> u64 { + let (a, b, c) = self; + a.heap_size_bytes() + b.heap_size_bytes() + c.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + T::is_pod() && U::is_pod() && V::is_pod() + } +} + +impl SizeBytes for (T, U, V, W) +where + T: SizeBytes, + U: SizeBytes, + V: SizeBytes, + W: SizeBytes, +{ + #[inline] + fn heap_size_bytes(&self) -> u64 { + let (a, b, c, d) = self; + a.heap_size_bytes() + b.heap_size_bytes() + c.heap_size_bytes() + d.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + T::is_pod() && U::is_pod() && V::is_pod() && W::is_pod() + } +} From ea04ea5ea279167f1104677155309d46f9733795 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 18:14:42 +0100 Subject: [PATCH 3/9] Implement sizes of arrow arrays --- .../re_types_core/src/size_bytes/arrow_sizes.rs | 17 +++++++++++++++++ .../store/re_types_core/src/size_bytes/mod.rs | 1 + 2 files changed, 18 insertions(+) create mode 100644 crates/store/re_types_core/src/size_bytes/arrow_sizes.rs diff --git a/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs b/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs new file mode 100644 index 000000000000..48e6da87d4cc --- /dev/null +++ b/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs @@ -0,0 +1,17 @@ +use arrow::array::{Array, ArrayRef}; + +use super::SizeBytes; + +impl SizeBytes for dyn Array { + #[inline] + fn heap_size_bytes(&self) -> u64 { + Box::::from(self).heap_size_bytes() + } +} + +impl SizeBytes for ArrayRef { + #[inline] + fn heap_size_bytes(&self) -> u64 { + Box::::from(self.as_ref()).heap_size_bytes() + } +} diff --git a/crates/store/re_types_core/src/size_bytes/mod.rs b/crates/store/re_types_core/src/size_bytes/mod.rs index c05a1bf1f5c4..a746ada41f39 100644 --- a/crates/store/re_types_core/src/size_bytes/mod.rs +++ b/crates/store/re_types_core/src/size_bytes/mod.rs @@ -1,4 +1,5 @@ mod arrow2_sizes; +mod arrow_sizes; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::sync::Arc; From 512a67175299adb27b5de304b6bf7100ff829c26 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 19:03:53 +0100 Subject: [PATCH 4/9] Less arrow2 --- Cargo.lock | 3 + crates/store/re_chunk/src/builder.rs | 18 +++++ crates/store/re_query/src/latest_at.rs | 13 +++- crates/viewer/re_component_ui/Cargo.toml | 1 + .../src/datatype_uis/singleline_string.rs | 9 ++- .../viewer/re_component_ui/src/fallback_ui.rs | 12 +++- .../re_data_ui/src/component_ui_registry.rs | 2 +- crates/viewer/re_selection_panel/Cargo.toml | 1 + .../re_selection_panel/src/defaults_ui.rs | 2 +- .../re_selection_panel/src/visualizer_ui.rs | 31 ++++---- crates/viewer/re_view/src/view_property_ui.rs | 4 +- crates/viewer/re_view_dataframe/Cargo.toml | 1 + .../src/display_record_batch.rs | 4 +- .../viewer/re_view_graph/src/ui/selection.rs | 5 +- .../src/visualizers/utilities/labels.rs | 4 +- .../re_viewer/src/blueprint/validation.rs | 2 +- .../src/blueprint_helpers.rs | 18 ++--- .../src/component_ui_registry.rs | 72 +++++++++---------- crates/viewer/re_viewer_context/src/item.rs | 2 +- .../src/view/view_context.rs | 3 +- .../viewer/re_viewport_blueprint/src/view.rs | 2 +- .../src/view_contents.rs | 4 +- examples/rust/extend_viewer_ui/src/main.rs | 2 +- 23 files changed, 119 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bf98eb72cf9..d1b14fa53311 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5701,6 +5701,7 @@ dependencies = [ name = "re_component_ui" version = "0.21.0-alpha.1+dev" dependencies = [ + "arrow", "egui", "egui_extras", "egui_plot", @@ -6297,6 +6298,7 @@ dependencies = [ name = "re_selection_panel" version = "0.21.0-alpha.1+dev" dependencies = [ + "arrow", "egui", "egui_tiles", "itertools 0.13.0", @@ -6597,6 +6599,7 @@ name = "re_view_dataframe" version = "0.21.0-alpha.1+dev" dependencies = [ "anyhow", + "arrow", "egui", "egui_table", "itertools 0.13.0", diff --git a/crates/store/re_chunk/src/builder.rs b/crates/store/re_chunk/src/builder.rs index 48ff0facdad4..33a0c449d950 100644 --- a/crates/store/re_chunk/src/builder.rs +++ b/crates/store/re_chunk/src/builder.rs @@ -1,3 +1,4 @@ +use arrow::array::ArrayRef; use arrow2::{ array::{Array as Arrow2Array, PrimitiveArray as Arrow2PrimitiveArray}, datatypes::DataType as Arrow2Datatype, @@ -99,6 +100,23 @@ impl ChunkBuilder { self } + /// Add a row's worth of data using the given component data. + #[inline] + pub fn with_row_arrow1( + self, + row_id: RowId, + timepoint: impl Into, + components: impl IntoIterator, + ) -> Self { + self.with_sparse_row( + row_id, + timepoint, + components + .into_iter() + .map(|(component_descr, array)| (component_descr, Some(array.into()))), + ) + } + /// Add a row's worth of data using the given component data. #[inline] pub fn with_row( diff --git a/crates/store/re_query/src/latest_at.rs b/crates/store/re_query/src/latest_at.rs index efc84879ae44..2bca910b2d12 100644 --- a/crates/store/re_query/src/latest_at.rs +++ b/crates/store/re_query/src/latest_at.rs @@ -12,7 +12,8 @@ use re_chunk::{Chunk, RowId, UnitChunkShared}; use re_chunk_store::{ChunkStore, LatestAtQuery, TimeInt}; use re_log_types::EntityPath; use re_types_core::{ - components::ClearIsRecursive, Component, ComponentDescriptor, ComponentName, SizeBytes, + components::ClearIsRecursive, external::arrow::array::ArrayRef, Component, ComponentDescriptor, + ComponentName, SizeBytes, }; use crate::{QueryCache, QueryCacheKey, QueryError}; @@ -313,7 +314,15 @@ impl LatestAtResults { /// Returns the raw data for the specified component. #[inline] - pub fn component_batch_raw( + pub fn component_batch_raw(&self, component_name: &ComponentName) -> Option { + self.components + .get(component_name)? + .component_batch_raw(component_name) + } + + /// Returns the raw data for the specified component. + #[inline] + pub fn component_batch_raw_arrow2( &self, component_name: &ComponentName, ) -> Option> { diff --git a/crates/viewer/re_component_ui/Cargo.toml b/crates/viewer/re_component_ui/Cargo.toml index eac5afe72e67..e77648689bab 100644 --- a/crates/viewer/re_component_ui/Cargo.toml +++ b/crates/viewer/re_component_ui/Cargo.toml @@ -32,6 +32,7 @@ re_types_core.workspace = true re_ui.workspace = true re_viewer_context.workspace = true +arrow.workspace = true egui_extras.workspace = true egui_plot.workspace = true egui.workspace = true diff --git a/crates/viewer/re_component_ui/src/datatype_uis/singleline_string.rs b/crates/viewer/re_component_ui/src/datatype_uis/singleline_string.rs index faa912c0f64c..510ca3755bfa 100644 --- a/crates/viewer/re_component_ui/src/datatype_uis/singleline_string.rs +++ b/crates/viewer/re_component_ui/src/datatype_uis/singleline_string.rs @@ -1,7 +1,6 @@ use re_types::{ components::{Name, Text}, datatypes::Utf8, - external::arrow2, Loggable as _, }; use re_ui::UiExt as _; @@ -84,9 +83,9 @@ pub fn display_text_ui( _db: &EntityDb, _path: &EntityPath, _row_id: Option, - data: &dyn arrow2::array::Array, + data: &dyn arrow::array::Array, ) { - let text = match Text::from_arrow2(data) { + let text = match Text::from_arrow(data) { Ok(text) => text.first().cloned(), Err(err) => { ui.error_label("Failed to deserialize") @@ -113,9 +112,9 @@ pub fn display_name_ui( _db: &EntityDb, _path: &EntityPath, _row_id: Option, - data: &dyn arrow2::array::Array, + data: &dyn arrow::array::Array, ) { - let name = match Name::from_arrow2(data) { + let name = match Name::from_arrow(data) { Ok(name) => name.first().cloned(), Err(err) => { ui.error_label("Failed to deserialize") diff --git a/crates/viewer/re_component_ui/src/fallback_ui.rs b/crates/viewer/re_component_ui/src/fallback_ui.rs index e392916fdf4b..73ddab380bea 100644 --- a/crates/viewer/re_component_ui/src/fallback_ui.rs +++ b/crates/viewer/re_component_ui/src/fallback_ui.rs @@ -17,12 +17,20 @@ pub fn fallback_component_ui( _db: &EntityDb, _entity_path: &EntityPath, _row_id: Option, - component: &dyn arrow2::array::Array, + component: &dyn arrow::array::Array, ) { arrow_ui(ui, ui_layout, component); } -fn arrow_ui(ui: &mut egui::Ui, ui_layout: UiLayout, array: &dyn arrow2::array::Array) { +fn arrow_ui(ui: &mut egui::Ui, ui_layout: UiLayout, array: &dyn arrow::array::Array) { + arrow2_ui( + ui, + ui_layout, + Box::::from(array).as_ref(), + ); +} + +fn arrow2_ui(ui: &mut egui::Ui, ui_layout: UiLayout, array: &dyn arrow2::array::Array) { use re_types::SizeBytes as _; // Special-treat text. diff --git a/crates/viewer/re_data_ui/src/component_ui_registry.rs b/crates/viewer/re_data_ui/src/component_ui_registry.rs index b0c4fab22e6d..b9218eaffaa0 100644 --- a/crates/viewer/re_data_ui/src/component_ui_registry.rs +++ b/crates/viewer/re_data_ui/src/component_ui_registry.rs @@ -23,7 +23,7 @@ pub fn add_to_registry(registry: &mut Com registry.add_legacy_display_ui( C::name(), Box::new( - |ctx, ui, ui_layout, query, db, entity_path, row_id, component_raw| match C::from_arrow2( + |ctx, ui, ui_layout, query, db, entity_path, row_id, component_raw| match C::from_arrow( component_raw, ) { Ok(components) => match components.len() { diff --git a/crates/viewer/re_selection_panel/Cargo.toml b/crates/viewer/re_selection_panel/Cargo.toml index aa09cf939c9a..3c55589ee298 100644 --- a/crates/viewer/re_selection_panel/Cargo.toml +++ b/crates/viewer/re_selection_panel/Cargo.toml @@ -35,6 +35,7 @@ re_ui.workspace = true re_viewer_context.workspace = true re_viewport_blueprint.workspace = true +arrow.workspace = true egui_tiles.workspace = true egui.workspace = true itertools.workspace = true diff --git a/crates/viewer/re_selection_panel/src/defaults_ui.rs b/crates/viewer/re_selection_panel/src/defaults_ui.rs index adcbadb7b77a..603a5301bede 100644 --- a/crates/viewer/re_selection_panel/src/defaults_ui.rs +++ b/crates/viewer/re_selection_panel/src/defaults_ui.rs @@ -201,7 +201,7 @@ fn active_defaults( db.storage_engine() .cache() .latest_at(query, &view.defaults_path, [*c]) - .component_batch_raw(c) + .component_batch_raw_arrow2(c) .map_or(false, |data| !data.is_empty()) }) .collect::>() diff --git a/crates/viewer/re_selection_panel/src/visualizer_ui.rs b/crates/viewer/re_selection_panel/src/visualizer_ui.rs index e8e04c99ad15..488da5d24e99 100644 --- a/crates/viewer/re_selection_panel/src/visualizer_ui.rs +++ b/crates/viewer/re_selection_panel/src/visualizer_ui.rs @@ -5,7 +5,7 @@ use re_data_ui::{sorted_component_list_for_ui, DataUi}; use re_entity_db::EntityDb; use re_log_types::{ComponentPath, EntityPath}; use re_types::blueprint::components::VisualizerOverrides; -use re_types::external::arrow2; +use re_types_core::external::arrow::array::ArrayRef; use re_ui::{list_item, UiExt as _}; use re_view::latest_at_with_blueprint_resolved_data; use re_viewer_context::{ @@ -160,9 +160,9 @@ fn visualizer_components( fn non_empty_component_batch_raw( unit: Option<&UnitChunkShared>, component_name: &ComponentName, - ) -> Option<(Option, Box)> { + ) -> Option<(Option, ArrayRef)> { let unit = unit?; - let batch = unit.component_batch_raw_arrow2(component_name)?; + let batch = unit.component_batch_raw(component_name)?; if batch.is_empty() { None } else { @@ -205,10 +205,9 @@ fn visualizer_components( let result_default = query_result.defaults.get(&component_name); let raw_default = non_empty_component_batch_raw(result_default, &component_name); - let raw_fallback: Box = visualizer + let raw_fallback = visualizer .fallback_provider() - .fallback_for(&query_ctx, component_name) - .into(); + .fallback_for(&query_ctx, component_name); // Determine where the final value comes from. // Putting this into an enum makes it easier to reason about the next steps. @@ -398,8 +397,8 @@ fn visualizer_components( override_path, &raw_override.clone().map(|(_, raw_override)| raw_override), raw_default.clone().map(|(_, raw_override)| raw_override), - raw_fallback.as_ref(), - raw_current_value.as_ref(), + raw_fallback.clone(), + raw_current_value.clone(), ); }), add_children, @@ -418,7 +417,7 @@ fn editable_blueprint_component_list_item( blueprint_path: &EntityPath, component: re_types::ComponentName, row_id: Option, - raw_override: &dyn arrow2::array::Array, + raw_override: &dyn arrow::array::Array, ) -> egui::Response { ui.list_item_flat_noninteractive( list_item::PropertyContent::new(name) @@ -450,10 +449,10 @@ fn menu_more( ui: &mut egui::Ui, component_name: re_types::ComponentName, override_path: &EntityPath, - raw_override: &Option>, - raw_default: Option>, - raw_fallback: &dyn arrow2::array::Array, - raw_current_value: &dyn arrow2::array::Array, + raw_override: &Option, + raw_default: Option, + raw_fallback: arrow::array::ArrayRef, + raw_current_value: arrow::array::ArrayRef, ) { if ui .add_enabled(raw_override.is_some(), egui::Button::new("Remove override")) @@ -479,14 +478,14 @@ fn menu_more( } if ui.button("Set to fallback value").clicked() { - ctx.save_blueprint_array(override_path, component_name, raw_fallback.to_boxed()); + ctx.save_blueprint_array(override_path, component_name, raw_fallback); ui.close_menu(); } let override_differs_from_default = raw_override != &ctx .viewer_ctx - .raw_latest_at_in_default_blueprint_arrow2(override_path, component_name); + .raw_latest_at_in_default_blueprint(override_path, component_name); if ui .add_enabled( override_differs_from_default, @@ -504,7 +503,7 @@ fn menu_more( ctx.save_blueprint_array( &ViewBlueprint::defaults_path(ctx.view_id), component_name, - raw_current_value.to_boxed(), + raw_current_value, ); ui.close_menu(); } diff --git a/crates/viewer/re_view/src/view_property_ui.rs b/crates/viewer/re_view/src/view_property_ui.rs index 1307c2d41ea4..995a3825803b 100644 --- a/crates/viewer/re_view/src/view_property_ui.rs +++ b/crates/viewer/re_view/src/view_property_ui.rs @@ -89,9 +89,7 @@ pub fn view_property_component_ui( field: &ArchetypeFieldReflection, fallback_provider: &dyn ComponentFallbackProvider, ) { - let component_array: Option> = property - .component_raw(field.component_name) - .map(|array| array.into()); + let component_array = property.component_raw(field.component_name); let row_id = property.component_row_id(field.component_name); let ui_types = ctx diff --git a/crates/viewer/re_view_dataframe/Cargo.toml b/crates/viewer/re_view_dataframe/Cargo.toml index 871c6cab85f9..425b6bd42e62 100644 --- a/crates/viewer/re_view_dataframe/Cargo.toml +++ b/crates/viewer/re_view_dataframe/Cargo.toml @@ -33,6 +33,7 @@ re_viewer_context.workspace = true re_viewport_blueprint.workspace = true anyhow.workspace = true +arrow.workspace = true egui_table.workspace = true egui.workspace = true itertools.workspace = true diff --git a/crates/viewer/re_view_dataframe/src/display_record_batch.rs b/crates/viewer/re_view_dataframe/src/display_record_batch.rs index e8217b0ca165..6260f74cb7e0 100644 --- a/crates/viewer/re_view_dataframe/src/display_record_batch.rs +++ b/crates/viewer/re_view_dataframe/src/display_record_batch.rs @@ -145,6 +145,8 @@ impl ComponentData { data }; + let data_to_display: arrow::array::ArrayRef = data_to_display.into(); + ctx.component_ui_registry.ui_raw( ctx, ui, @@ -154,7 +156,7 @@ impl ComponentData { entity_path, component_name, None, - &*data_to_display, + data_to_display.as_ref(), ); } else { ui.label("-"); diff --git a/crates/viewer/re_view_graph/src/ui/selection.rs b/crates/viewer/re_view_graph/src/ui/selection.rs index 659e4dbb120e..d8b8906051bc 100644 --- a/crates/viewer/re_view_graph/src/ui/selection.rs +++ b/crates/viewer/re_view_graph/src/ui/selection.rs @@ -1,4 +1,3 @@ -use re_chunk::Arrow2Array; use re_types::{ blueprint::components::Enabled, Archetype, ArchetypeReflectionMarker, Component as _, }; @@ -62,9 +61,7 @@ pub fn view_property_force_ui( .find(|field| field.component_name == Enabled::name()) .expect("forces are required to have an `Enabled` component"); - let component_array: Option> = property - .component_raw(field.component_name) - .map(|array| array.into()); + let component_array = property.component_raw(field.component_name); let row_id = property.component_row_id(field.component_name); let singleline_ui: &dyn Fn(&mut egui::Ui) = &|ui| { diff --git a/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs b/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs index ea72d531b5b4..6b238c651666 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs @@ -115,10 +115,10 @@ pub fn show_labels_fallback(ctx: &re_viewer_context::QueryContext< ctx.recording() .latest_at(ctx.query, ctx.target_entity_path, [C::name(), Text::name()]); let num_instances = results - .component_batch_raw(&C::name()) + .component_batch_raw_arrow2(&C::name()) .map_or(0, |array| array.len()); let num_labels = results - .component_batch_raw(&Text::name()) + .component_batch_raw_arrow2(&Text::name()) .map_or(0, |array| array.len()); ShowLabels::from(num_labels == 1 || num_instances < MAX_NUM_LABELS_PER_ENTITY) diff --git a/crates/viewer/re_viewer/src/blueprint/validation.rs b/crates/viewer/re_viewer/src/blueprint/validation.rs index 048c4569e2ec..4735289919f6 100644 --- a/crates/viewer/re_viewer/src/blueprint/validation.rs +++ b/crates/viewer/re_viewer/src/blueprint/validation.rs @@ -24,7 +24,7 @@ pub(crate) fn validate_component(blueprint: &EntityDb) -> bool { if let Some(array) = engine .cache() .latest_at(&query, path, [C::name()]) - .component_batch_raw(&C::name()) + .component_batch_raw_arrow2(&C::name()) { if let Err(err) = C::from_arrow2_opt(&*array) { re_log::debug!( diff --git a/crates/viewer/re_viewer_context/src/blueprint_helpers.rs b/crates/viewer/re_viewer_context/src/blueprint_helpers.rs index c3912a3da785..bcf84dd327d8 100644 --- a/crates/viewer/re_viewer_context/src/blueprint_helpers.rs +++ b/crates/viewer/re_viewer_context/src/blueprint_helpers.rs @@ -1,5 +1,5 @@ use arrow::array::ArrayRef; -use re_chunk::{Arrow2Array, RowId}; +use re_chunk::RowId; use re_chunk_store::external::re_chunk::Chunk; use re_log_types::{EntityPath, TimeInt, TimePoint, Timeline}; use re_types::{AsComponents, ComponentBatch, ComponentDescriptor, ComponentName}; @@ -87,12 +87,12 @@ impl ViewerContext<'_> { &self, entity_path: &EntityPath, component_name: ComponentName, - array: Box, + array: ArrayRef, ) { let timepoint = self.store_context.blueprint_timepoint_for_writes(); let chunk = match Chunk::builder(entity_path.clone()) - .with_row( + .with_row_arrow1( RowId::new(), timepoint.clone(), [(ComponentDescriptor::new(component_name), array)], @@ -138,16 +138,6 @@ impl ViewerContext<'_> { }) } - /// Queries a raw component from the default blueprint. - pub fn raw_latest_at_in_default_blueprint_arrow2( - &self, - entity_path: &EntityPath, - component_name: ComponentName, - ) -> Option> { - self.raw_latest_at_in_default_blueprint(entity_path, component_name) - .map(|array| array.into()) - } - /// Resets a blueprint component to the value it had in the default blueprint. pub fn reset_blueprint_component_by_name( &self, @@ -155,7 +145,7 @@ impl ViewerContext<'_> { component_name: ComponentName, ) { if let Some(default_value) = - self.raw_latest_at_in_default_blueprint_arrow2(entity_path, component_name) + self.raw_latest_at_in_default_blueprint(entity_path, component_name) { self.save_blueprint_array(entity_path, component_name, default_value); } else { diff --git a/crates/viewer/re_viewer_context/src/component_ui_registry.rs b/crates/viewer/re_viewer_context/src/component_ui_registry.rs index 1e93023de52e..e04caa265280 100644 --- a/crates/viewer/re_viewer_context/src/component_ui_registry.rs +++ b/crates/viewer/re_viewer_context/src/component_ui_registry.rs @@ -1,14 +1,11 @@ use std::collections::BTreeMap; -use re_chunk::{Arrow2Array, RowId, UnitChunkShared}; +use re_chunk::{RowId, UnitChunkShared}; use re_chunk_store::LatestAtQuery; use re_entity_db::{EntityDb, EntityPath}; use re_log::ResultExt; use re_log_types::Instance; -use re_types::{ - external::arrow2::{self}, - ComponentName, -}; +use re_types::ComponentName; use re_ui::UiExt as _; use crate::{ComponentFallbackProvider, MaybeMutRef, QueryContext, ViewerContext}; @@ -174,7 +171,7 @@ type LegacyDisplayComponentUiCallback = Box< &EntityDb, &EntityPath, Option, - &dyn arrow2::array::Array, + &dyn arrow::array::Array, ) + Send + Sync, >; @@ -196,9 +193,9 @@ type UntypedComponentEditOrViewCallback = Box< dyn Fn( &ViewerContext<'_>, &mut egui::Ui, - &dyn arrow2::array::Array, + &dyn arrow::array::Array, EditOrView, - ) -> Option> + ) -> Option + Send + Sync, >; @@ -336,7 +333,7 @@ impl ComponentUiRegistry { if response.changed() { use re_types::LoggableBatch as _; - deserialized_value.to_arrow2().ok_or_log_error_once() + deserialized_value.to_arrow().ok_or_log_error_once() } else { None } @@ -391,7 +388,7 @@ impl ComponentUiRegistry { // Don't use component.raw_instance here since we want to handle the case where there's several // elements differently. // Also, it allows us to slice the array without cloning any elements. - let Some(array) = unit.component_batch_raw_arrow2(&component_name) else { + let Some(array) = unit.component_batch_raw(&component_name) else { re_log::error_once!("Couldn't get {component_name}: missing"); ui.error_with_details_on_hover(format!("Couldn't get {component_name}: missing")); return; @@ -413,7 +410,7 @@ impl ComponentUiRegistry { // Enforce clamp-to-border semantics. // TODO(andreas): Is that always what we want? let index = index.clamp(0, array.len().saturating_sub(1)); - let component_raw = array.sliced(index, 1); + let component_raw = array.slice(index, 1); self.ui_raw( ctx, @@ -440,7 +437,7 @@ impl ComponentUiRegistry { entity_path: &EntityPath, component_name: ComponentName, row_id: Option, - component_raw: &dyn arrow2::array::Array, + component_raw: &dyn arrow::array::Array, ) { re_tracing::profile_function!(component_name.full_name()); @@ -504,7 +501,7 @@ impl ComponentUiRegistry { blueprint_write_path: &EntityPath, component_name: ComponentName, row_id: Option, - component_array: Option<&dyn Arrow2Array>, + component_array: Option<&dyn arrow::array::Array>, fallback_provider: &dyn ComponentFallbackProvider, ) { let multiline = true; @@ -535,7 +532,7 @@ impl ComponentUiRegistry { blueprint_write_path: &EntityPath, component_name: ComponentName, row_id: Option, - component_query_result: Option<&dyn Arrow2Array>, + component_query_result: Option<&dyn arrow::array::Array>, fallback_provider: &dyn ComponentFallbackProvider, ) { let multiline = false; @@ -561,31 +558,32 @@ impl ComponentUiRegistry { blueprint_write_path: &EntityPath, component_name: ComponentName, row_id: Option, - component_array: Option<&dyn Arrow2Array>, + component_array: Option<&dyn arrow::array::Array>, fallback_provider: &dyn ComponentFallbackProvider, allow_multiline: bool, ) { re_tracing::profile_function!(component_name.full_name()); - // Use a fallback if there's either no component data at all or the component array is empty. - let component_raw = if let Some(component_raw) = - component_array.and_then(|array| (!array.is_empty()).then(|| array.to_boxed())) - { - component_raw - } else { - fallback_provider.fallback_for(ctx, component_name).into() + let mut run_with = |array| { + self.edit_ui_raw( + ctx, + ui, + origin_db, + blueprint_write_path, + component_name, + row_id, + array, + allow_multiline, + ); }; - self.edit_ui_raw( - ctx, - ui, - origin_db, - blueprint_write_path, - component_name, - row_id, - component_raw.as_ref(), - allow_multiline, - ); + // Use a fallback if there's either no component data at all or the component array is empty. + if let Some(component_array) = component_array.filter(|array| !array.is_empty()) { + run_with(component_array); + } else { + let fallback = fallback_provider.fallback_for(ctx, component_name); + run_with(fallback.as_ref()); + } } #[allow(clippy::too_many_arguments)] @@ -597,13 +595,13 @@ impl ComponentUiRegistry { blueprint_write_path: &EntityPath, component_name: ComponentName, row_id: Option, - component_raw: &dyn arrow2::array::Array, + component_raw: &dyn arrow::array::Array, allow_multiline: bool, ) { if !self.try_show_edit_ui( ctx.viewer_ctx, ui, - component_raw.as_ref(), + component_raw, blueprint_write_path, component_name, allow_multiline, @@ -631,7 +629,7 @@ impl ComponentUiRegistry { &self, ctx: &ViewerContext<'_>, ui: &mut egui::Ui, - raw_current_value: &dyn arrow2::array::Array, + raw_current_value: &dyn arrow::array::Array, blueprint_write_path: &EntityPath, component_name: ComponentName, allow_multiline: bool, @@ -660,9 +658,9 @@ impl ComponentUiRegistry { } } -fn try_deserialize(value: &dyn arrow2::array::Array) -> Option { +fn try_deserialize(value: &dyn arrow::array::Array) -> Option { let component_name = C::name(); - let deserialized = C::from_arrow2(value); + let deserialized = C::from_arrow(value); match deserialized { Ok(values) => { if values.len() > 1 { diff --git a/crates/viewer/re_viewer_context/src/item.rs b/crates/viewer/re_viewer_context/src/item.rs index 6dec4f817629..7b0ebb2646ef 100644 --- a/crates/viewer/re_viewer_context/src/item.rs +++ b/crates/viewer/re_viewer_context/src/item.rs @@ -211,7 +211,7 @@ pub fn resolve_mono_instance_path( &instance.entity_path, [ComponentDescriptor::new(component_name)], ) - .component_batch_raw(&component_name) + .component_batch_raw_arrow2(&component_name) { if array.len() > 1 { return instance.clone(); diff --git a/crates/viewer/re_viewer_context/src/view/view_context.rs b/crates/viewer/re_viewer_context/src/view/view_context.rs index 9f8104552328..be33c854ba4d 100644 --- a/crates/viewer/re_viewer_context/src/view/view_context.rs +++ b/crates/viewer/re_viewer_context/src/view/view_context.rs @@ -1,6 +1,5 @@ use std::sync::Arc; -use re_chunk::Arrow2Array; use re_chunk_store::LatestAtQuery; use re_log_types::{EntityPath, TimePoint}; use re_query::StorageEngineReadGuard; @@ -97,7 +96,7 @@ impl<'a> ViewContext<'a> { &self, entity_path: &EntityPath, component_name: ComponentName, - array: Box, + array: arrow::array::ArrayRef, ) { self.viewer_ctx .save_blueprint_array(entity_path, component_name, array); diff --git a/crates/viewer/re_viewport_blueprint/src/view.rs b/crates/viewer/re_viewport_blueprint/src/view.rs index 3177beae78da..44390a462fe6 100644 --- a/crates/viewer/re_viewport_blueprint/src/view.rs +++ b/crates/viewer/re_viewport_blueprint/src/view.rs @@ -263,7 +263,7 @@ impl ViewBlueprint { let array = blueprint_engine .cache() .latest_at(query, path, [component_name]) - .component_batch_raw(&component_name); + .component_batch_raw_arrow2(&component_name); array.map(|array| (ComponentDescriptor::new(component_name), array)) }), ) diff --git a/crates/viewer/re_viewport_blueprint/src/view_contents.rs b/crates/viewer/re_viewport_blueprint/src/view_contents.rs index 7988a7c9d65a..534cd3982c8f 100644 --- a/crates/viewer/re_viewport_blueprint/src/view_contents.rs +++ b/crates/viewer/re_viewport_blueprint/src/view_contents.rs @@ -491,7 +491,7 @@ impl DataQueryPropertyResolver<'_> { .storage_engine() .cache() .latest_at(blueprint_query, &recursive_override_path, [component_name]) - .component_batch_raw(&component_name) + .component_batch_raw_arrow2(&component_name) { if !component_data.is_empty() { recursive_property_overrides.to_mut().insert( @@ -522,7 +522,7 @@ impl DataQueryPropertyResolver<'_> { .storage_engine() .cache() .latest_at(blueprint_query, &individual_override_path, [component_name]) - .component_batch_raw(&component_name) + .component_batch_raw_arrow2(&component_name) { if !component_data.is_empty() { resolved_component_overrides.insert( diff --git a/examples/rust/extend_viewer_ui/src/main.rs b/examples/rust/extend_viewer_ui/src/main.rs index 76724ef27de2..073ec3e24e1e 100644 --- a/examples/rust/extend_viewer_ui/src/main.rs +++ b/examples/rust/extend_viewer_ui/src/main.rs @@ -163,7 +163,7 @@ fn component_ui( .cache() .latest_at(&query, entity_path, [component_name]); - if let Some(data) = results.component_batch_raw(&component_name) { + if let Some(data) = results.component_batch_raw_arrow2(&component_name) { egui::ScrollArea::vertical() .auto_shrink([false, true]) .show(ui, |ui| { From 9bd7052331ac163afbde835e8dbb464c83426402 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 19:09:56 +0100 Subject: [PATCH 5/9] Less arrow2 --- crates/store/re_chunk/src/builder.rs | 8 ++++---- crates/store/re_chunk/src/lib.rs | 1 + crates/store/re_chunk/src/merge.rs | 4 ++-- crates/viewer/re_data_ui/src/component.rs | 2 +- crates/viewer/re_selection_panel/src/defaults_ui.rs | 7 ++----- .../re_view_spatial/src/visualizers/utilities/labels.rs | 4 ++-- crates/viewer/re_viewer/src/blueprint/validation.rs | 4 ++-- crates/viewer/re_viewer_context/src/blueprint_helpers.rs | 6 +++--- crates/viewer/re_viewer_context/src/item.rs | 2 +- crates/viewer/re_viewport_blueprint/src/view.rs | 2 +- crates/viewer/re_viewport_blueprint/src/view_contents.rs | 4 ++-- examples/rust/extend_viewer_ui/src/main.rs | 4 ++-- 12 files changed, 23 insertions(+), 25 deletions(-) diff --git a/crates/store/re_chunk/src/builder.rs b/crates/store/re_chunk/src/builder.rs index 33a0c449d950..8834e673d7c8 100644 --- a/crates/store/re_chunk/src/builder.rs +++ b/crates/store/re_chunk/src/builder.rs @@ -102,7 +102,7 @@ impl ChunkBuilder { /// Add a row's worth of data using the given component data. #[inline] - pub fn with_row_arrow1( + pub fn with_row( self, row_id: RowId, timepoint: impl Into, @@ -119,7 +119,7 @@ impl ChunkBuilder { /// Add a row's worth of data using the given component data. #[inline] - pub fn with_row( + pub fn with_row_arrow2( self, row_id: RowId, timepoint: impl Into, @@ -160,7 +160,7 @@ impl ChunkBuilder { timepoint: impl Into, component_batch: &dyn ComponentBatch, ) -> Self { - self.with_row( + self.with_row_arrow2( row_id, timepoint, component_batch @@ -178,7 +178,7 @@ impl ChunkBuilder { timepoint: impl Into, component_batches: impl IntoIterator, ) -> Self { - self.with_row( + self.with_row_arrow2( row_id, timepoint, component_batches.into_iter().filter_map(|component_batch| { diff --git a/crates/store/re_chunk/src/lib.rs b/crates/store/re_chunk/src/lib.rs index 159b47d44ca5..564885c20101 100644 --- a/crates/store/re_chunk/src/lib.rs +++ b/crates/store/re_chunk/src/lib.rs @@ -48,6 +48,7 @@ pub use re_log_types::{EntityPath, TimeInt, TimePoint, Timeline, TimelineName}; pub use re_types_core::{ArchetypeFieldName, ArchetypeName, ComponentName}; pub mod external { + pub use arrow; pub use arrow2; pub use nohash_hasher; diff --git a/crates/store/re_chunk/src/merge.rs b/crates/store/re_chunk/src/merge.rs index e61467ab69c2..1777e94655cd 100644 --- a/crates/store/re_chunk/src/merge.rs +++ b/crates/store/re_chunk/src/merge.rs @@ -804,7 +804,7 @@ mod tests { ::to_arrow2(&MyPoint64::new(1.0, 1.0))?; let chunk1 = Chunk::builder(entity_path.into()) - .with_row( + .with_row_arrow2( row_id1, timepoint1, [ @@ -814,7 +814,7 @@ mod tests { .build()?; let chunk2 = Chunk::builder(entity_path.into()) - .with_row( + .with_row_arrow2( row_id2, timepoint2, [ diff --git a/crates/viewer/re_data_ui/src/component.rs b/crates/viewer/re_data_ui/src/component.rs index c49af8d75113..663923ddb5cd 100644 --- a/crates/viewer/re_data_ui/src/component.rs +++ b/crates/viewer/re_data_ui/src/component.rs @@ -33,7 +33,7 @@ impl DataUi for ComponentPathLatestAtResults<'_> { let Some(num_instances) = self .unit - .component_batch_raw_arrow2(component_name) + .component_batch_raw(component_name) .map(|data| data.len()) else { ui.weak(""); diff --git a/crates/viewer/re_selection_panel/src/defaults_ui.rs b/crates/viewer/re_selection_panel/src/defaults_ui.rs index 603a5301bede..1800b68ca702 100644 --- a/crates/viewer/re_selection_panel/src/defaults_ui.rs +++ b/crates/viewer/re_selection_panel/src/defaults_ui.rs @@ -201,7 +201,7 @@ fn active_defaults( db.storage_engine() .cache() .latest_at(query, &view.defaults_path, [*c]) - .component_batch_raw_arrow2(c) + .component_batch_raw(c) .map_or(false, |data| !data.is_empty()) }) .collect::>() @@ -304,10 +304,7 @@ fn add_popup_ui( .with_row( RowId::new(), ctx.blueprint_timepoint_for_writes(), - [( - ComponentDescriptor::new(component_name), - initial_data.into(), - )], + [(ComponentDescriptor::new(component_name), initial_data)], ) .build() { diff --git a/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs b/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs index 6b238c651666..ea72d531b5b4 100644 --- a/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs +++ b/crates/viewer/re_view_spatial/src/visualizers/utilities/labels.rs @@ -115,10 +115,10 @@ pub fn show_labels_fallback(ctx: &re_viewer_context::QueryContext< ctx.recording() .latest_at(ctx.query, ctx.target_entity_path, [C::name(), Text::name()]); let num_instances = results - .component_batch_raw_arrow2(&C::name()) + .component_batch_raw(&C::name()) .map_or(0, |array| array.len()); let num_labels = results - .component_batch_raw_arrow2(&Text::name()) + .component_batch_raw(&Text::name()) .map_or(0, |array| array.len()); ShowLabels::from(num_labels == 1 || num_instances < MAX_NUM_LABELS_PER_ENTITY) diff --git a/crates/viewer/re_viewer/src/blueprint/validation.rs b/crates/viewer/re_viewer/src/blueprint/validation.rs index 4735289919f6..2d5c10a1db2f 100644 --- a/crates/viewer/re_viewer/src/blueprint/validation.rs +++ b/crates/viewer/re_viewer/src/blueprint/validation.rs @@ -24,9 +24,9 @@ pub(crate) fn validate_component(blueprint: &EntityDb) -> bool { if let Some(array) = engine .cache() .latest_at(&query, path, [C::name()]) - .component_batch_raw_arrow2(&C::name()) + .component_batch_raw(&C::name()) { - if let Err(err) = C::from_arrow2_opt(&*array) { + if let Err(err) = C::from_arrow_opt(&*array) { re_log::debug!( "Failed to deserialize component {:?}: {:?}", C::name(), diff --git a/crates/viewer/re_viewer_context/src/blueprint_helpers.rs b/crates/viewer/re_viewer_context/src/blueprint_helpers.rs index bcf84dd327d8..ab3a16bda1de 100644 --- a/crates/viewer/re_viewer_context/src/blueprint_helpers.rs +++ b/crates/viewer/re_viewer_context/src/blueprint_helpers.rs @@ -92,7 +92,7 @@ impl ViewerContext<'_> { let timepoint = self.store_context.blueprint_timepoint_for_writes(); let chunk = match Chunk::builder(entity_path.clone()) - .with_row_arrow1( + .with_row( RowId::new(), timepoint.clone(), [(ComponentDescriptor::new(component_name), array)], @@ -165,7 +165,7 @@ impl ViewerContext<'_> { .latest_at(self.blueprint_query, entity_path, [component_name]) .get(&component_name) .and_then(|unit| { - unit.component_batch_raw_arrow2(&component_name) + unit.component_batch_raw(&component_name) .map(|array| array.data_type().clone()) }) else { @@ -180,7 +180,7 @@ impl ViewerContext<'_> { timepoint, [( ComponentDescriptor::new(component_name), - re_chunk::external::arrow2::array::new_empty_array(datatype), + re_chunk::external::arrow::array::new_empty_array(&datatype), )], ) .build(); diff --git a/crates/viewer/re_viewer_context/src/item.rs b/crates/viewer/re_viewer_context/src/item.rs index 7b0ebb2646ef..6dec4f817629 100644 --- a/crates/viewer/re_viewer_context/src/item.rs +++ b/crates/viewer/re_viewer_context/src/item.rs @@ -211,7 +211,7 @@ pub fn resolve_mono_instance_path( &instance.entity_path, [ComponentDescriptor::new(component_name)], ) - .component_batch_raw_arrow2(&component_name) + .component_batch_raw(&component_name) { if array.len() > 1 { return instance.clone(); diff --git a/crates/viewer/re_viewport_blueprint/src/view.rs b/crates/viewer/re_viewport_blueprint/src/view.rs index 44390a462fe6..3177beae78da 100644 --- a/crates/viewer/re_viewport_blueprint/src/view.rs +++ b/crates/viewer/re_viewport_blueprint/src/view.rs @@ -263,7 +263,7 @@ impl ViewBlueprint { let array = blueprint_engine .cache() .latest_at(query, path, [component_name]) - .component_batch_raw_arrow2(&component_name); + .component_batch_raw(&component_name); array.map(|array| (ComponentDescriptor::new(component_name), array)) }), ) diff --git a/crates/viewer/re_viewport_blueprint/src/view_contents.rs b/crates/viewer/re_viewport_blueprint/src/view_contents.rs index 534cd3982c8f..7988a7c9d65a 100644 --- a/crates/viewer/re_viewport_blueprint/src/view_contents.rs +++ b/crates/viewer/re_viewport_blueprint/src/view_contents.rs @@ -491,7 +491,7 @@ impl DataQueryPropertyResolver<'_> { .storage_engine() .cache() .latest_at(blueprint_query, &recursive_override_path, [component_name]) - .component_batch_raw_arrow2(&component_name) + .component_batch_raw(&component_name) { if !component_data.is_empty() { recursive_property_overrides.to_mut().insert( @@ -522,7 +522,7 @@ impl DataQueryPropertyResolver<'_> { .storage_engine() .cache() .latest_at(blueprint_query, &individual_override_path, [component_name]) - .component_batch_raw_arrow2(&component_name) + .component_batch_raw(&component_name) { if !component_data.is_empty() { resolved_component_overrides.insert( diff --git a/examples/rust/extend_viewer_ui/src/main.rs b/examples/rust/extend_viewer_ui/src/main.rs index 073ec3e24e1e..7c8cb9918133 100644 --- a/examples/rust/extend_viewer_ui/src/main.rs +++ b/examples/rust/extend_viewer_ui/src/main.rs @@ -171,13 +171,13 @@ fn component_ui( let num_instances = data.len(); for i in 0..num_instances { - ui.label(format_arrow(&*data.sliced(i, 1))); + ui.label(format_arrow2(&*data.sliced(i, 1))); } }); }; } -fn format_arrow(value: &dyn arrow2::array::Array) -> String { +fn format_arrow2(value: &dyn arrow2::array::Array) -> String { use re_types::SizeBytes as _; let bytes = value.total_size_bytes(); From f3b5a7b150fff13bb0ccc5c5ebbe7cdd15697cc0 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 19:16:41 +0100 Subject: [PATCH 6/9] More conversion; better naming --- crates/viewer/re_chunk_store_ui/src/arrow_ui.rs | 2 +- crates/viewer/re_chunk_store_ui/src/chunk_ui.rs | 2 +- .../re_view_time_series/src/line_visualizer_system.rs | 10 +++++----- .../re_view_time_series/src/point_visualizer_system.rs | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs b/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs index aee17cbd5442..f3ee0ef8abd3 100644 --- a/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs +++ b/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs @@ -8,7 +8,7 @@ use re_ui::UiExt; // Note: this is copied and heavily modified from `re_data_ui`. We don't want to unify them because // that would likely introduce an undesired dependency (`re_chunk_store_ui` should remain as // independent as possible from the viewer, so it may be split off one day). -pub(crate) fn arrow_ui(ui: &mut egui::Ui, array: &dyn arrow2::array::Array) { +pub(crate) fn arrow2_ui(ui: &mut egui::Ui, array: &dyn arrow2::array::Array) { ui.scope(|ui| { ui.style_mut().wrap_mode = Some(egui::TextWrapMode::Truncate); diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs b/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs index 2007531b7512..d7283f48a6a5 100644 --- a/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs +++ b/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs @@ -149,7 +149,7 @@ impl ChunkUi { chunk.component_batch_raw(&component_desc.component_name, row_index); match component_data { Some(Ok(data)) => { - crate::arrow_ui::arrow_ui(ui, &*data); + crate::arrow_ui::arrow2_ui(ui, &*data); } Some(Err(err)) => { ui.error_with_details_on_hover(err.to_string()); diff --git a/crates/viewer/re_view_time_series/src/line_visualizer_system.rs b/crates/viewer/re_view_time_series/src/line_visualizer_system.rs index 140bc073bf28..03ee2a16469b 100644 --- a/crates/viewer/re_view_time_series/src/line_visualizer_system.rs +++ b/crates/viewer/re_view_time_series/src/line_visualizer_system.rs @@ -4,7 +4,7 @@ use re_chunk_store::{RangeQuery, RowId}; use re_log_types::{EntityPath, TimeInt}; use re_types::archetypes; use re_types::components::{AggregationPolicy, ClearIsRecursive}; -use re_types::external::arrow2::datatypes::DataType as Arrow2Datatype; +use re_types::external::arrow::datatypes::DataType as ArrowDatatype; use re_types::{ archetypes::SeriesLine, components::{Color, Name, Scalar, StrokeWidth}, @@ -243,7 +243,7 @@ impl SeriesLineSystem { chunk.iter_component_indices(&query.timeline(), &Scalar::name()) }) .map(|(data_time, _)| { - debug_assert_eq!(Scalar::arrow2_datatype(), Arrow2Datatype::Float64); + debug_assert_eq!(Scalar::arrow_datatype(), ArrowDatatype::Float64); PlotPoint { time: data_time.as_i64(), @@ -257,7 +257,7 @@ impl SeriesLineSystem { { re_tracing::profile_scope!("fill values"); - debug_assert_eq!(Scalar::arrow2_datatype(), Arrow2Datatype::Float64); + debug_assert_eq!(Scalar::arrow_datatype(), ArrowDatatype::Float64); all_scalar_chunks .iter() .flat_map(|chunk| chunk.iter_primitive::(&Scalar::name())) @@ -281,7 +281,7 @@ impl SeriesLineSystem { { re_tracing::profile_scope!("fill colors"); - debug_assert_eq!(Color::arrow2_datatype(), Arrow2Datatype::UInt32); + debug_assert_eq!(Color::arrow_datatype(), ArrowDatatype::UInt32); fn map_raw_color(raw: &[u32]) -> Option { raw.first().map(|c| { @@ -334,7 +334,7 @@ impl SeriesLineSystem { { re_tracing::profile_scope!("fill stroke widths"); - debug_assert_eq!(StrokeWidth::arrow2_datatype(), Arrow2Datatype::Float32); + debug_assert_eq!(StrokeWidth::arrow_datatype(), ArrowDatatype::Float32); { let all_stroke_width_chunks = results.get_optional_chunks(&StrokeWidth::name()); diff --git a/crates/viewer/re_view_time_series/src/point_visualizer_system.rs b/crates/viewer/re_view_time_series/src/point_visualizer_system.rs index 9ab73b3855f1..912fbd149474 100644 --- a/crates/viewer/re_view_time_series/src/point_visualizer_system.rs +++ b/crates/viewer/re_view_time_series/src/point_visualizer_system.rs @@ -3,7 +3,7 @@ use itertools::Itertools as _; use re_types::{ archetypes::{self, SeriesPoint}, components::{Color, MarkerShape, MarkerSize, Name, Scalar}, - external::arrow2::datatypes::DataType as Arrow2Datatype, + external::arrow::datatypes::DataType as ArrowDatatype, Archetype as _, Component as _, Loggable as _, }; use re_view::range_with_blueprint_resolved_data; @@ -257,7 +257,7 @@ impl SeriesPointSystem { chunk.iter_component_indices(&query.timeline(), &Scalar::name()) }) .map(|(data_time, _)| { - debug_assert_eq!(Scalar::arrow2_datatype(), Arrow2Datatype::Float64); + debug_assert_eq!(Scalar::arrow_datatype(), ArrowDatatype::Float64); PlotPoint { time: data_time.as_i64(), @@ -271,7 +271,7 @@ impl SeriesPointSystem { { re_tracing::profile_scope!("fill values"); - debug_assert_eq!(Scalar::arrow2_datatype(), Arrow2Datatype::Float64); + debug_assert_eq!(Scalar::arrow_datatype(), ArrowDatatype::Float64); let mut i = 0; all_scalar_chunks .iter() @@ -297,7 +297,7 @@ impl SeriesPointSystem { { re_tracing::profile_scope!("fill colors"); - debug_assert_eq!(Color::arrow2_datatype(), Arrow2Datatype::UInt32); + debug_assert_eq!(Color::arrow_datatype(), ArrowDatatype::UInt32); fn map_raw_color(raw: &[u32]) -> Option { raw.first().map(|c| { @@ -351,7 +351,7 @@ impl SeriesPointSystem { { re_tracing::profile_scope!("fill marker sizes"); - debug_assert_eq!(MarkerSize::arrow2_datatype(), Arrow2Datatype::Float32); + debug_assert_eq!(MarkerSize::arrow_datatype(), ArrowDatatype::Float32); { let all_marker_size_chunks = results.get_optional_chunks(&MarkerSize::name()); From 32616358223c2579de1376e0d5ea3a8406d051f8 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 19:21:18 +0100 Subject: [PATCH 7/9] build-fix --- crates/store/re_types/src/lib.rs | 1 + crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/store/re_types/src/lib.rs b/crates/store/re_types/src/lib.rs index 9ad5c1d25cf2..3e75dcfa6fd4 100644 --- a/crates/store/re_types/src/lib.rs +++ b/crates/store/re_types/src/lib.rs @@ -277,6 +277,7 @@ pub mod external { pub use re_types_core; pub use anyhow; + pub use arrow; pub use arrow2; pub use ndarray; pub use uuid; diff --git a/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs b/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs index 6039c4bdcc4a..65e89b104e60 100644 --- a/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs +++ b/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs @@ -275,6 +275,7 @@ fn test_arrow2_estimated_size_bytes() { offset::Offsets, }; use std::mem::size_of; + use std::sync::Arc; // empty primitive array { From 052ef890d53aba7d3ab68e81c11d9cf08992025f Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 18 Dec 2024 20:16:25 +0100 Subject: [PATCH 8/9] Fix docstring --- crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs b/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs index 65e89b104e60..5a4b5f3a1b15 100644 --- a/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs +++ b/crates/store/re_types_core/src/size_bytes/arrow2_sizes.rs @@ -1,8 +1,7 @@ //! --- Arrow2 size estimations --- //! -//! The following is a modified version of [1], available under MIT OR Apache-2.0. -//! -//! [1] +//! The following is a modified version of , +//! available under MIT OR Apache-2.0. use std::collections::BTreeMap; From 8b59d3a450a70fc7827cd012b1222897e53633f9 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 19 Dec 2024 13:57:16 +0100 Subject: [PATCH 9/9] Use `get_array_memory_size` --- crates/store/re_types_core/src/size_bytes/arrow_sizes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs b/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs index 48e6da87d4cc..7b9ca72d0fd0 100644 --- a/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs +++ b/crates/store/re_types_core/src/size_bytes/arrow_sizes.rs @@ -5,13 +5,13 @@ use super::SizeBytes; impl SizeBytes for dyn Array { #[inline] fn heap_size_bytes(&self) -> u64 { - Box::::from(self).heap_size_bytes() + self.get_array_memory_size() as u64 } } impl SizeBytes for ArrayRef { #[inline] fn heap_size_bytes(&self) -> u64 { - Box::::from(self.as_ref()).heap_size_bytes() + self.get_array_memory_size() as u64 } }