Skip to content

Commit

Permalink
end-to-end batching 1: introduce DataCell & retire ComponentBundle (
Browse files Browse the repository at this point in the history 
#1634)

* introduce DataCell, retire ComponentBundle

* oh. my. god.

* other misc improvements while we're here...

* addressing PR comments

* link to 1712

* auto rename failed me

* addressing PR comments
  • Loading branch information
@teh-cmc
teh-cmc authored Mar 27, 2023
1 parent 1cd2bb5 commit 17a21bf
Show file tree
Hide file tree
Showing 45 changed files with 928 additions and 898 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,17 @@ ctrlc = { version = "3.0", features = ["termination"] }
ecolor = "0.21.0"
eframe = { version = "0.21.3", default-features = false }
egui = "0.21.0"
egui-wgpu = "0.21.0"
egui_dock = "0.4"
egui_extras = "0.21.0"
egui-wgpu = "0.21.0"
emath = "0.21.0"
enumset = "1.0.12"
epaint = "0.21.0"
glam = "0.22"
gltf = "1.1"
half = "2.0"
image = { version = "0.24", default-features = false }
itertools = "0.10"
lazy_static = "1.4"
macaw = "0.18"
mimalloc = "0.1.29"
Expand Down
2 changes: 1 addition & 1 deletion crates/re_arrow_store/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ arrow2 = { workspace = true, features = [
] }
document-features = "0.2"
indent = "0.1"
itertools = "0.10"
itertools = { workspace = true }
nohash-hasher = "0.2"
parking_lot.workspace = true
static_assertions = "1.1"
Expand Down
2 changes: 1 addition & 1 deletion crates/re_arrow_store/benches/data_store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ fn insert_messages<'a>(
msgs: impl Iterator<Item = &'a MsgBundle>,
) -> DataStore {
let mut store = DataStore::new(cluster_key, config);
msgs.for_each(|msg_bundle| store.insert(msg_bundle).unwrap());
msgs.for_each(|msg_bundle| store.insert_row(msg_bundle).unwrap());
store
}

Expand Down
18 changes: 9 additions & 9 deletions crates/re_arrow_store/examples/dump_dataframe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,48 +29,48 @@ fn main() {
let bundle1 = test_bundle!(ent_path @ [
build_frame_nr(1.into()), build_log_time(Time::now()),
] => [build_some_instances(2), build_some_rects(2)]);
store.insert(&bundle1).unwrap();
store.insert_row(&bundle1).unwrap();
}

for ent_path in &ent_paths {
let bundle2 = test_bundle!(ent_path @ [
build_frame_nr(2.into())
] => [build_some_instances(2), build_some_point2d(2)]);
store.insert(&bundle2).unwrap();
store.insert_row(&bundle2).unwrap();
// Insert timelessly too!
let bundle2 =
test_bundle!(ent_path @ [] => [build_some_instances(2), build_some_point2d(2)]);
store.insert(&bundle2).unwrap();
store.insert_row(&bundle2).unwrap();

let bundle3 = test_bundle!(ent_path @ [
build_frame_nr(3.into()), build_log_time(Time::now()),
] => [build_some_instances_from(25..29), build_some_point2d(4)]);
store.insert(&bundle3).unwrap();
store.insert_row(&bundle3).unwrap();
// Insert timelessly too!
let bundle3 = test_bundle!(ent_path @ [] => [build_some_instances_from(25..29), build_some_point2d(4)]);
store.insert(&bundle3).unwrap();
store.insert_row(&bundle3).unwrap();
}

for ent_path in &ent_paths {
let bundle4_1 = test_bundle!(ent_path @ [
build_frame_nr(4.into()), build_log_time(Time::now()),
] => [build_some_instances_from(20..23), build_some_rects(3)]);
store.insert(&bundle4_1).unwrap();
store.insert_row(&bundle4_1).unwrap();

let bundle4_15 = test_bundle!(ent_path @ [
build_frame_nr(4.into()),
] => [build_some_instances_from(20..23), build_some_point2d(3)]);
store.insert(&bundle4_15).unwrap();
store.insert_row(&bundle4_15).unwrap();

let bundle4_2 = test_bundle!(ent_path @ [
build_frame_nr(4.into()), build_log_time(Time::now()),
] => [build_some_instances_from(25..28), build_some_rects(3)]);
store.insert(&bundle4_2).unwrap();
store.insert_row(&bundle4_2).unwrap();

let bundle4_25 = test_bundle!(ent_path @ [
build_frame_nr(4.into()), build_log_time(Time::now()),
] => [build_some_instances_from(25..28), build_some_point2d(3)]);
store.insert(&bundle4_25).unwrap();
store.insert_row(&bundle4_25).unwrap();
}

let df = store.to_dataframe();
Expand Down
4 changes: 2 additions & 2 deletions crates/re_arrow_store/examples/latest_component.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ fn main() {
let ent_path = EntityPath::from("my/entity");

let bundle = test_bundle!(ent_path @ [build_frame_nr(2.into())] => [build_some_rects(4)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(3.into())] => [build_some_point2d(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);

Expand Down
4 changes: 2 additions & 2 deletions crates/re_arrow_store/examples/latest_components.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ fn main() {
let ent_path = EntityPath::from("my/entity");

let bundle = test_bundle!(ent_path @ [build_frame_nr(2.into())] => [build_some_rects(4)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(3.into())] => [build_some_point2d(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);
let df = latest_components(
Expand Down
14 changes: 7 additions & 7 deletions crates/re_arrow_store/examples/range_components.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,25 +24,25 @@ fn main() {
let frame4 = 4.into();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame1)] => [build_some_rects(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame2)] => [build_some_point2d(2)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame3)] => [build_some_point2d(4)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_rects(3)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_point2d(1)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_rects(3)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let bundle = test_bundle!(ent_path @ [build_frame_nr(frame4)] => [build_some_point2d(3)]);
store.insert(&bundle).unwrap();
store.insert_row(&bundle).unwrap();

let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);
let query = RangeQuery::new(timeline_frame_nr, TimeRange::new(2.into(), 4.into()));
Expand Down
Loading

1 comment on commit 17a21bf

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rust Benchmark

Benchmark suite Current: 17a21bf Previous: 1cd2bb5 Ratio
datastore/insert/batch/rects/insert 590043 ns/iter (± 5508) 547858 ns/iter (± 2567) 1.08
datastore/latest_at/batch/rects/query 1860 ns/iter (± 8) 1837 ns/iter (± 63) 1.01
datastore/latest_at/missing_components/primary 286 ns/iter (± 3) 286 ns/iter (± 2) 1
datastore/latest_at/missing_components/secondaries 449 ns/iter (± 5) 436 ns/iter (± 4) 1.03
datastore/range/batch/rects/query 151042 ns/iter (± 1473) 149977 ns/iter (± 1640) 1.01
mono_points_arrow/generate_message_bundles 29095634 ns/iter (± 1333541) 44892991 ns/iter (± 1287024) 0.65
mono_points_arrow/generate_messages 119014936 ns/iter (± 1374794) 123888941 ns/iter (± 1228014) 0.96
mono_points_arrow/encode_log_msg 154653153 ns/iter (± 1118379) 155425399 ns/iter (± 919285) 1.00
mono_points_arrow/encode_total 304640531 ns/iter (± 2118477) 326384994 ns/iter (± 2070978) 0.93
mono_points_arrow/decode_log_msg 173545782 ns/iter (± 1309046) 173943214 ns/iter (± 1431381) 1.00
mono_points_arrow/decode_message_bundles 52990283 ns/iter (± 1041855) 62214762 ns/iter (± 869818) 0.85
mono_points_arrow/decode_total 223851542 ns/iter (± 1851584) 236709065 ns/iter (± 1901027) 0.95
batch_points_arrow/generate_message_bundles 314821 ns/iter (± 2630) 338605 ns/iter (± 3591) 0.93
batch_points_arrow/generate_messages 5892 ns/iter (± 59) 6239 ns/iter (± 55) 0.94
batch_points_arrow/encode_log_msg 348381 ns/iter (± 3122) 368388 ns/iter (± 2211) 0.95
batch_points_arrow/encode_total 693811 ns/iter (± 7159) 727145 ns/iter (± 4398) 0.95
batch_points_arrow/decode_log_msg 344026 ns/iter (± 2478) 345949 ns/iter (± 1799) 0.99
batch_points_arrow/decode_message_bundles 1553 ns/iter (± 18) 2007 ns/iter (± 13) 0.77
batch_points_arrow/decode_total 351462 ns/iter (± 2125) 354295 ns/iter (± 1875) 0.99
arrow_mono_points/insert 6240768409 ns/iter (± 19864832) 6038756633 ns/iter (± 15898319) 1.03
arrow_mono_points/query 1759378 ns/iter (± 16492) 1769635 ns/iter (± 18790) 0.99
arrow_batch_points/insert 3060805 ns/iter (± 22787) 2693496 ns/iter (± 12107) 1.14
arrow_batch_points/query 16418 ns/iter (± 151) 15981 ns/iter (± 130) 1.03
arrow_batch_vecs/insert 44053 ns/iter (± 323) 42337 ns/iter (± 313) 1.04
arrow_batch_vecs/query 385099 ns/iter (± 4018) 385290 ns/iter (± 2919) 1.00
tuid/Tuid::random 34 ns/iter (± 0) 34 ns/iter (± 0) 1

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.