diff --git a/crates/re_data_store/tests/data_store.rs b/crates/re_data_store/tests/data_store.rs index 6d13079d9e4f..66fd981e5304 100644 --- a/crates/re_data_store/tests/data_store.rs +++ b/crates/re_data_store/tests/data_store.rs @@ -8,7 +8,7 @@ use re_data_store::{ test_row, test_util::{init_logs, insert_table_with_retries, sanity_unwrap}, DataStore, DataStoreConfig, DataStoreStats, GarbageCollectionOptions, GarbageCollectionTarget, - LatestAtQuery, TimeInt, TimeRange, + LatestAtQuery, RangeQuery, TimeInt, TimeRange, }; use re_log_types::{build_frame_nr, DataRow, DataTable, EntityPath, TableId, TimeType, Timeline}; use re_types::datagen::{ @@ -378,8 +378,6 @@ fn latest_at_impl(store: &mut DataStore) { // --- Range --- -// TODO: requires join semantics -> re_query - #[test] fn range() { init_logs(); @@ -478,282 +476,117 @@ fn range_impl(store: &mut DataStore) { } let store = store2; - // TODO - // let mut expected_timeless = Vec::::new(); - // let mut expected_at_times: IntMap> = Default::default(); - // - // for (time, rows) in rows_at_times { - // if let Some(time) = time { - // let dfs = expected_at_times.entry(*time).or_default(); - // dfs.push(joint_df(store.cluster_key(), rows)); - // } else { - // expected_timeless.push(joint_df(store.cluster_key(), rows)); - // } - // } - // - // let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence); - // - // store.sort_indices_if_needed(); // for assertions below - // - // let components = [InstanceKey::name(), components[0], components[1]]; - // let query = RangeQuery::new(timeline_frame_nr, time_range); - // let dfs = polars_util::range_components( - // &store, - // &query, - // &ent_path, - // components[1], - // components, - // &JoinType::Outer, - // ); - // - // let mut dfs_processed = 0usize; - // let mut timeless_count = 0usize; - // let mut time_counters: IntMap = Default::default(); - // for (time, df) in dfs.map(Result::unwrap) { - // let df_expected = if let Some(time) = time { - // let time_count = time_counters.entry(time.as_i64()).or_default(); - // let df_expected = &expected_at_times[&time][*time_count]; - // *time_count += 1; - // df_expected - // } else { - // let df_expected = &expected_timeless[timeless_count]; - // timeless_count += 1; - // df_expected - // }; - // - // assert_eq!(*df_expected, df, "{store}"); - // - // dfs_processed += 1; - // } - // - // let dfs_processed_expected = rows_at_times.len(); - // assert_eq!(dfs_processed_expected, dfs_processed); + let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence); + + store.sort_indices_if_needed(); // for assertions below + + let components = [components[0], components[1]]; + let query = RangeQuery::new(timeline_frame_nr, time_range); + let results = store.range(&query, &ent_path, components); + + let mut results_processed = 0usize; + for (i, (time, _, cells)) in results.enumerate() { + let (expected_time, expected_rows) = rows_at_times[i]; + assert_eq!(expected_time, time); + + for (component_name, expected) in expected_rows { + let expected = expected + .cells + .iter() + .filter(|cell| cell.component_name() == *component_name) + .collect_vec(); + let actual = cells.iter().flatten().collect_vec(); + assert_eq!(expected, actual); + + results_processed += 1; + } + } + + let results_processed_expected = rows_at_times.len(); + assert_eq!(results_processed_expected, results_processed); }; // TODO(cmc): bring back some log_time scenarios - // Unit ranges (Color's PoV) + // Unit ranges (multi-PoV) assert_range_components( TimeRange::new(frame1, frame1), [Color::name(), Position2D::name()], &[ - ( - Some(frame1), - &[ - (Color::name(), &row1), - (Position2D::name(), &row4_4), // timeless - ], - ), // + (Some(frame1), &[(Color::name(), &row1)]), // ], ); assert_range_components( TimeRange::new(frame2, frame2), [Color::name(), Position2D::name()], - &[], - ); - assert_range_components( - TimeRange::new(frame3, frame3), - [Color::name(), Position2D::name()], - &[], - ); - assert_range_components( - TimeRange::new(frame4, frame4), - [Color::name(), Position2D::name()], &[ - ( - Some(frame4), - &[(Color::name(), &row4_1), (Position2D::name(), &row3)], - ), - ( - Some(frame4), - &[(Color::name(), &row4_2), (Position2D::name(), &row3)], - ), - ( - Some(frame4), - &[(Color::name(), &row4_3), (Position2D::name(), &row4_25)], // !!! - ), - ], - ); - assert_range_components( - TimeRange::new(frame5, frame5), - [Color::name(), Position2D::name()], - &[], - ); - - // Unit ranges (Position2D's PoV) - - assert_range_components( - TimeRange::new(frame1, frame1), - [Position2D::name(), Color::name()], - &[], - ); - assert_range_components( - TimeRange::new(frame2, frame2), - [Position2D::name(), Color::name()], - &[ - ( - Some(frame2), - &[(Position2D::name(), &row2), (Color::name(), &row1)], - ), // + (Some(frame2), &[(Position2D::name(), &row2)]), // ], ); assert_range_components( TimeRange::new(frame3, frame3), - [Position2D::name(), Color::name()], + [Color::name(), Position2D::name()], &[ - ( - Some(frame3), - &[(Position2D::name(), &row3), (Color::name(), &row1)], - ), // + (Some(frame3), &[(Position2D::name(), &row3)]), // ], ); assert_range_components( TimeRange::new(frame4, frame4), - [Position2D::name(), Color::name()], + [Color::name(), Position2D::name()], &[ - ( - Some(frame4), - &[(Position2D::name(), &row4_25), (Color::name(), &row4_2)], - ), - ( - Some(frame4), - &[(Position2D::name(), &row4_4), (Color::name(), &row4_3)], - ), + (Some(frame4), &[(Color::name(), &row4_1)]), + (Some(frame4), &[(Color::name(), &row4_2)]), + (Some(frame4), &[(Position2D::name(), &row4_25)]), + (Some(frame4), &[(Color::name(), &row4_3)]), + (Some(frame4), &[(Position2D::name(), &row4_4)]), ], ); assert_range_components( TimeRange::new(frame5, frame5), - [Position2D::name(), Color::name()], + [Color::name(), Position2D::name()], &[], ); - // Full range (Color's PoV) + // Full range (multi-PoV) assert_range_components( TimeRange::new(frame1, frame5), [Color::name(), Position2D::name()], &[ - ( - Some(frame1), - &[ - (Color::name(), &row1), - (Position2D::name(), &row4_4), // timeless - ], - ), - ( - Some(frame4), - &[(Color::name(), &row4_1), (Position2D::name(), &row3)], - ), - ( - Some(frame4), - &[(Color::name(), &row4_2), (Position2D::name(), &row3)], - ), - ( - Some(frame4), - &[(Color::name(), &row4_3), (Position2D::name(), &row4_25)], // !!! - ), + (Some(frame1), &[(Color::name(), &row1)]), // + (Some(frame2), &[(Position2D::name(), &row2)]), // + (Some(frame3), &[(Position2D::name(), &row3)]), // + (Some(frame4), &[(Color::name(), &row4_1)]), + (Some(frame4), &[(Color::name(), &row4_2)]), + (Some(frame4), &[(Position2D::name(), &row4_25)]), + (Some(frame4), &[(Color::name(), &row4_3)]), + (Some(frame4), &[(Position2D::name(), &row4_4)]), ], ); - // Full range (Position2D's PoV) - - assert_range_components( - TimeRange::new(frame1, frame5), - [Position2D::name(), Color::name()], - &[ - ( - Some(frame2), - &[(Position2D::name(), &row2), (Color::name(), &row1)], - ), - ( - Some(frame3), - &[(Position2D::name(), &row3), (Color::name(), &row1)], - ), - ( - Some(frame4), - &[(Position2D::name(), &row4_25), (Color::name(), &row4_2)], - ), - ( - Some(frame4), - &[(Position2D::name(), &row4_4), (Color::name(), &row4_3)], - ), - ], - ); - - // Infinite range (Color's PoV) + // Infinite range (multi-PoV) assert_range_components( TimeRange::new(TimeInt::MIN, TimeInt::MAX), [Color::name(), Position2D::name()], &[ - (None, &[(Color::name(), &row1)]), - ( - None, - &[(Color::name(), &row4_1), (Position2D::name(), &row3)], - ), - ( - None, - &[(Color::name(), &row4_2), (Position2D::name(), &row3)], - ), - ( - None, - &[(Color::name(), &row4_3), (Position2D::name(), &row4_25)], // !!! - ), - ( - Some(frame1), - &[ - (Color::name(), &row1), - (Position2D::name(), &row4_4), // timeless - ], - ), - ( - Some(frame4), - &[(Color::name(), &row4_1), (Position2D::name(), &row3)], - ), - ( - Some(frame4), - &[(Color::name(), &row4_2), (Position2D::name(), &row3)], - ), - ( - Some(frame4), - &[(Color::name(), &row4_3), (Position2D::name(), &row4_25)], // !!! - ), - ], - ); - - // Infinite range (Position2D's PoV) - - assert_range_components( - TimeRange::new(TimeInt::MIN, TimeInt::MAX), - [Position2D::name(), Color::name()], - &[ - (None, &[(Position2D::name(), &row2), (Color::name(), &row1)]), - (None, &[(Position2D::name(), &row3), (Color::name(), &row1)]), - ( - None, - &[(Position2D::name(), &row4_25), (Color::name(), &row4_2)], - ), - ( - None, - &[(Position2D::name(), &row4_4), (Color::name(), &row4_3)], - ), - ( - Some(frame2), - &[(Position2D::name(), &row2), (Color::name(), &row1)], - ), - ( - Some(frame3), - &[(Position2D::name(), &row3), (Color::name(), &row1)], - ), - ( - Some(frame4), - &[(Position2D::name(), &row4_25), (Color::name(), &row4_2)], - ), - ( - Some(frame4), - &[(Position2D::name(), &row4_4), (Color::name(), &row4_3)], - ), + (None, &[(Color::name(), &row1)]), // + (None, &[(Position2D::name(), &row2)]), // + (None, &[(Position2D::name(), &row3)]), // + (None, &[(Color::name(), &row4_1)]), + (None, &[(Color::name(), &row4_2)]), + (None, &[(Position2D::name(), &row4_25)]), + (None, &[(Color::name(), &row4_3)]), + (None, &[(Position2D::name(), &row4_4)]), + (Some(frame1), &[(Color::name(), &row1)]), // + (Some(frame2), &[(Position2D::name(), &row2)]), // + (Some(frame3), &[(Position2D::name(), &row3)]), // + (Some(frame4), &[(Color::name(), &row4_1)]), + (Some(frame4), &[(Color::name(), &row4_2)]), + (Some(frame4), &[(Position2D::name(), &row4_25)]), + (Some(frame4), &[(Color::name(), &row4_3)]), + (Some(frame4), &[(Position2D::name(), &row4_4)]), ], ); } diff --git a/crates/re_query/tests/store.rs b/crates/re_query/tests/store.rs index f5e72b0df61d..1aec304a1b36 100644 --- a/crates/re_query/tests/store.rs +++ b/crates/re_query/tests/store.rs @@ -1,173 +1,14 @@ -use std::collections::HashMap; - use itertools::Itertools; -use re_data_store::{ - test_row, - test_util::{insert_table_with_retries, sanity_unwrap}, - DataStore, GarbageCollectionOptions, LatestAtQuery, -}; +use re_data_store::{test_row, DataStore}; use re_log_types::EntityPath; -use re_log_types::{build_frame_nr, DataRow, DataTable, TableId, TimeInt, TimeType, Timeline}; +use re_log_types::{build_frame_nr, TimeInt, TimeType, Timeline}; use re_types::{ archetypes::Points2D, components::{Color, InstanceKey, Position2D}, - datagen::{build_some_colors, build_some_instances, build_some_positions2d}, + datagen::{build_some_colors, build_some_positions2d}, }; -use re_types_core::{Archetype, Component, ComponentName, Loggable as _}; - -// --- LatestAt --- - -// TODO: at this point, why is this even here? - -#[test] -fn latest_at() { - for config in re_data_store::test_util::all_configs() { - let mut store = DataStore::new( - re_log_types::StoreId::random(re_log_types::StoreKind::Recording), - InstanceKey::name(), - config.clone(), - ); - latest_at_impl(&mut store); - } -} - -fn latest_at_impl(store: &mut DataStore) { - let ent_path = EntityPath::from("this/that"); - - let frame0 = TimeInt::from(0); - let frame1 = TimeInt::from(1); - let frame2 = TimeInt::from(2); - let frame3 = TimeInt::from(3); - let frame4 = TimeInt::from(4); - - // helper to insert a table both as a temporal and timeless payload - let insert_table = |store: &mut DataStore, table: &DataTable| { - // insert temporal - insert_table_with_retries(store, table); - - // insert timeless - let mut table_timeless = table.clone(); - table_timeless.col_timelines = Default::default(); - insert_table_with_retries(store, &table_timeless); - }; - - let (instances1, colors1) = (build_some_instances(3), build_some_colors(3)); - let row1 = test_row!(ent_path @ [build_frame_nr(frame1)] => 3; [instances1.clone(), colors1]); - - let positions2 = build_some_positions2d(3); - let row2 = test_row!(ent_path @ [build_frame_nr(frame2)] => 3; [instances1, positions2]); - - let points3 = build_some_positions2d(10); - let row3 = test_row!(ent_path @ [build_frame_nr(frame3)] => 10; [points3]); - - let colors4 = build_some_colors(5); - let row4 = test_row!(ent_path @ [build_frame_nr(frame4)] => 5; [colors4]); - - insert_table( - store, - &DataTable::from_rows( - TableId::new(), - [row1.clone(), row2.clone(), row3.clone(), row4.clone()], - ), - ); - - // Stress test save-to-disk & load-from-disk - let mut store2 = DataStore::new( - re_log_types::StoreId::random(re_log_types::StoreKind::Recording), - store.cluster_key(), - store.config().clone(), - ); - for table in store.to_data_tables(None) { - insert_table(&mut store2, &table); - } - // Stress test GC - store2.gc(&GarbageCollectionOptions::gc_everything()); - for table in store.to_data_tables(None) { - insert_table(&mut store2, &table); - } - let store = store2; - - sanity_unwrap(&store); - - fn assert_latest_components< - A: Archetype, - P1: Component + PartialEq + std::fmt::Debug, - C1: Component + PartialEq + std::fmt::Debug, - >( - store: &DataStore, - entity_path: &EntityPath, - frame_nr: TimeInt, - rows: &[(ComponentName, &DataRow)], - ) { - let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence); - - let arch_view = re_query::query_archetype::( - store, - &LatestAtQuery::new(timeline_frame_nr, frame_nr), - entity_path, - ) - .unwrap(); - - let expected_p1 = arch_view - .iter_required_component::() - .unwrap() - .collect_vec(); - let expected_c1 = arch_view - .iter_raw_optional_component::() - .unwrap() - .unwrap() - .collect_vec(); - - let rows: HashMap = rows.iter().copied().collect(); - - let actual_p1 = { - let cells = &rows[&P1::name()].cells; - cells.last().unwrap().to_native::() - }; - let actual_c1 = { - let cells = &rows[&C1::name()].cells; - cells.last().unwrap().to_native::() - }; - - assert_eq!(expected_p1, actual_p1); - assert_eq!(expected_c1, actual_c1); - } - - assert_latest_components::( - &store, - &ent_path, - frame0, - &[(Color::name(), &row4), (Position2D::name(), &row3)], // timeless - ); - assert_latest_components::( - &store, - &ent_path, - frame1, - &[ - (Color::name(), &row1), - (Position2D::name(), &row3), // timeless - ], - ); - assert_latest_components::( - &store, - &ent_path, - frame2, - &[(Color::name(), &row1), (Position2D::name(), &row2)], - ); - assert_latest_components::( - &store, - &ent_path, - frame3, - &[(Color::name(), &row1), (Position2D::name(), &row3)], - ); - assert_latest_components::( - &store, - &ent_path, - frame4, - &[(Color::name(), &row4), (Position2D::name(), &row3)], - ); -} +use re_types_core::Loggable as _; // ---