Unify prototype data_store with existing code (#9)

Cargo.toml

@@ -2,9 +2,10 @@
 resolver = "2"
 members = [
     "comms",
+    "data_store",
     "log_types",
     "objectron",
-    "prototype",
+    "rr_string_interner",
     "viewer",
     "web_server",
 ]

prototype/Cargo.toml → data_store/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "prototype"
+name = "data_store"
 version = "0.1.0"
 edition = "2021"
 rust-version = "1.60"
@@ -8,6 +8,8 @@ publish = false
 
 
 [dependencies]
+log_types = { path = "../log_types" }
+
 ahash = "0.7"
 im = "15"
 mimalloc = "0.1"

prototype/benches/data_benchmark.rs → data_store/benches/data_benchmark.rs

@@ -2,8 +2,8 @@
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 use criterion::{criterion_group, criterion_main, Criterion};
-use prototype::TypePathDataStore;
-use prototype::*;
+use data_store::TypePathDataStore;
+use data_store::*;
 
 use std::sync::Arc;
 
@@ -12,13 +12,13 @@ const NUM_POINTS_PER_CAMERA: u64 = 1_000;
 const TOTAL_POINTS: u64 = 2 * NUM_POINTS_PER_CAMERA;
 
 fn data_path(camera: &str, index: u64, field: &str) -> DataPath {
-    im::vector![
-        DataPathComponent::Name("camera".into()),
+    DataPath(vec![
+        DataPathComponent::String("camera".into()),
         DataPathComponent::Index(Index::String(camera.into())),
-        DataPathComponent::Name("point".into()),
+        DataPathComponent::String("point".into()),
         DataPathComponent::Index(Index::Sequence(index)),
-        DataPathComponent::Name(field.into()),
-    ]
+        DataPathComponent::String(field.into()),
+    ])
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
@@ -44,11 +44,11 @@ fn generate_date(individual_pos: bool, individual_radius: bool) -> TypePathDataS
                 }
             } else {
                 let type_path = im::vector![
-                    TypePathComponent::Name("camera".into()),
+                    TypePathComponent::String("camera".into()),
                     TypePathComponent::Index,
-                    TypePathComponent::Name("point".into()),
+                    TypePathComponent::String("point".into()),
                     TypePathComponent::Index,
-                    TypePathComponent::Name("pos".into())
+                    TypePathComponent::String("pos".into())
                 ];
                 let mut index_path_prefix = IndexPathKey::default();
                 index_path_prefix.push_back(Index::String(camera.into()));
@@ -77,11 +77,11 @@ fn generate_date(individual_pos: bool, individual_radius: bool) -> TypePathDataS
                 }
             } else {
                 let type_path = im::vector![
-                    TypePathComponent::Name("camera".into()),
+                    TypePathComponent::String("camera".into()),
                     TypePathComponent::Index,
-                    TypePathComponent::Name("point".into()),
+                    TypePathComponent::String("point".into()),
                     TypePathComponent::Index,
-                    TypePathComponent::Name("radius".into())
+                    TypePathComponent::String("radius".into())
                 ];
                 let mut index_path_prefix = IndexPathKey::default();
                 index_path_prefix.push_back(Index::String(camera.into()));
@@ -106,12 +106,20 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("query-points-throughput");
     group.throughput(criterion::Throughput::Elements(TOTAL_POINTS as _));
 
+    let point_type_path = im::vector![
+        TypePathComponent::String("camera".into()),
+        TypePathComponent::Index,
+        TypePathComponent::String("point".into()),
+        TypePathComponent::Index,
+    ];
+
     let data_store = generate_date(false, false);
     group.bench_function("batched_pos_batched_radius", |b| {
         b.iter(|| {
             let scene =
                 Scene3D::from_store(&data_store, &TimeQuery::LatestAt(Time(NUM_FRAMES / 2)));
-            assert_eq!(scene.points.len(), TOTAL_POINTS as usize);
+            assert_eq!(scene.points.len(), 1);
+            assert_eq!(scene.points[&point_type_path].len(), TOTAL_POINTS as usize);
         });
     });
 
@@ -120,7 +128,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let scene =
                 Scene3D::from_store(&data_store, &TimeQuery::LatestAt(Time(NUM_FRAMES / 2)));
-            assert_eq!(scene.points.len(), TOTAL_POINTS as usize);
+            assert_eq!(scene.points.len(), 1);
+            assert_eq!(scene.points[&point_type_path].len(), TOTAL_POINTS as usize);
         });
     });
 
@@ -129,7 +138,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let scene =
                 Scene3D::from_store(&data_store, &TimeQuery::LatestAt(Time(NUM_FRAMES / 2)));
-            assert_eq!(scene.points.len(), TOTAL_POINTS as usize);
+            assert_eq!(scene.points.len(), 1);
+            assert_eq!(scene.points[&point_type_path].len(), TOTAL_POINTS as usize);
         });
     });
 
@@ -138,7 +148,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         b.iter(|| {
             let scene =
                 Scene3D::from_store(&data_store, &TimeQuery::LatestAt(Time(NUM_FRAMES / 2)));
-            assert_eq!(scene.points.len(), TOTAL_POINTS as usize);
+            assert_eq!(scene.points.len(), 1);
+            assert_eq!(scene.points[&point_type_path].len(), TOTAL_POINTS as usize);
         });
     });
 

prototype/examples/memory_usage.rs → data_store/examples/memory_usage.rs

@@ -44,7 +44,7 @@ unsafe impl std::alloc::GlobalAlloc for TrackingAllocator {
     }
 }
 
-use prototype::*;
+use data_store::*;
 
 impl TrackingAllocator {
     fn used_bytes(&self) -> usize {
@@ -53,13 +53,13 @@ impl TrackingAllocator {
 }
 
 fn data_path(camera: u64, index: u64, field: &str) -> DataPath {
-    im::vector![
-        DataPathComponent::Name("camera".into()),
+    DataPath(vec![
+        DataPathComponent::String("camera".into()),
         DataPathComponent::Index(Index::Sequence(camera)),
-        DataPathComponent::Name("point".into()),
+        DataPathComponent::String("point".into()),
         DataPathComponent::Index(Index::Sequence(index)),
-        DataPathComponent::Name(field.into()),
-    ]
+        DataPathComponent::String(field.into()),
+    ])
 }
 
 const BYTES_PER_POINT: usize = 16 + 24; // IndexPathKey + [f32; 3]

prototype/src/lib.rs → data_store/src/lib.rs

@@ -4,6 +4,8 @@ mod storage;
 pub use scene::*;
 pub use storage::*;
 
+pub use log_types::{DataPath, DataPathComponent, Index, TypePath, TypePathComponent};
+
 use std::collections::BTreeMap;
 
 pub enum AtomType {
@@ -50,29 +52,13 @@ pub enum StructType {
     Point3D,
 }
 
-// pub struct TypePath(Vec<TypePathComponent>);
-pub type TypePath = im::Vector<TypePathComponent>;
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum TypePathComponent {
-    /// Struct member
-    Name(String),
-
-    /// Table (array/map) member.
-    /// Tables are homogenous, so it is the same type path for all.
-    Index,
-}
-
-// pub struct DataPath(Vec<DataPathComponent>);
-pub type DataPath = im::Vector<DataPathComponent>;
-
 pub fn into_type_path(data_path: DataPath) -> (TypePath, IndexPathKey) {
     let mut type_path = im::Vector::default();
     let mut index_path = IndexPathKey::default();
     for component in data_path {
         match component {
-            DataPathComponent::Name(name) => {
-                type_path.push_back(TypePathComponent::Name(name));
+            DataPathComponent::String(name) => {
+                type_path.push_back(TypePathComponent::String(name));
             }
             DataPathComponent::Index(index) => {
                 type_path.push_back(TypePathComponent::Index);
@@ -83,33 +69,6 @@ pub fn into_type_path(data_path: DataPath) -> (TypePath, IndexPathKey) {
     (type_path, index_path)
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum DataPathComponent {
-    /// struct member
-    Name(String),
-
-    /// array/table/map member
-    Index(Index),
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub enum Index {
-    /// For arrays, assumed to be dense (0, 1, 2, …)
-    Sequence(u64),
-
-    /// X,Y pixel coordinates, from top left.
-    Pixel([u64; 2]),
-
-    /// Any integer, e.g. a hash
-    Integer(i128),
-
-    /// UUID/GUID
-    // Uuid(Uuid),
-
-    /// Anything goes
-    String(String),
-}
-
 // ----------------------------------------------------------------------------
 
 /// Like `Index` but also includes a precomputed hash.

prototype/src/scene.rs → data_store/src/scene.rs

@@ -1,14 +1,15 @@
 use crate::*;
 
-#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
+#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
 pub struct Point3<'s> {
     pub pos: &'s [f32; 3],
     pub radius: Option<f32>,
 }
 
 #[derive(Default)]
 pub struct Scene3D<'s> {
-    pub points: Vec<Point3<'s>>,
+    /// The path is the parent path, e.g. `points[43]` containing `points[43].pos`, `points[43].radius` etc.
+    pub points: BTreeMap<TypePath, Vec<Point3<'s>>>,
 }
 
 impl<'s> Scene3D<'s> {
@@ -19,22 +20,30 @@ impl<'s> Scene3D<'s> {
         let mut slf = Self::default();
 
         for (type_path, _) in store.iter() {
-            if type_path.last() == Some(&TypePathComponent::Name("pos".into())) {
+            if type_path.last() == Some(&TypePathComponent::String("pos".into())) {
+                let mut points = vec![];
                 visit_data_and_siblings(
                     store,
                     time_query,
                     type_path,
                     ("radius",),
                     |pos: &[f32; 3], radius: Option<&f32>| {
-                        slf.points.push(Point3 {
+                        points.push(Point3 {
                             pos,
                             radius: radius.copied(),
                         });
                     },
                 );
+                slf.points.insert(parent(type_path), points);
             }
         }
 
         slf
     }
 }
+
+fn parent(type_path: &TypePath) -> TypePath {
+    let mut type_path = type_path.clone();
+    type_path.pop_back();
+    type_path
+}