refactor: move generated google APIs to seperate crate

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "googleapis"]
-	path = googleapis
+	path = crates/googleapis/include
 	url = https://github.com/googleapis/googleapis.git

Cargo.toml

@@ -1,3 +1,9 @@
+[workspace]
+
+[workspace.dependencies]
+tokio = { version = "1.36.0" }
+tonic = { version = "0.11", features = ["gzip"] }
+
 [package]
 name = "firestore-emulator"
 version = "0.1.0"
@@ -22,24 +28,20 @@ color-eyre = "0.6.2"
 console = { version = "0.15.8", optional = true }
 console-subscriber = { version = "0.2.0", optional = true }
 futures = "0.3.30"
+googleapis = { path = "crates/googleapis" }
 itertools = "0.12.1"
-prost = "0.12"
-prost-types = "0.12"
 string_cache = "0.8.7"
 tikv-jemallocator = "0.5.4"
 time = { version = "0.3.34", features = [
     "formatting",
     "macros",
 ], optional = true }
-tokio = { version = "1.36.0", features = ["full"] }
+tokio = { workspace = true, features = ["full"] }
 tokio-stream = { version = "0.1.14", features = ["full"] }
-tonic = { version = "0.11", features = ["gzip"] }
+tonic = { workspace = true }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", optional = true }
 
-[build-dependencies]
-tonic-build = "0.11"
-
 [profile.release]
 codegen-unit = 1
 lto = "fat"

TODO.md

@@ -0,0 +1,6 @@
+- common-document-store-backend
+  - UNIMPLEMENTED: run_aggregation_query is not supported yet
+- wao-claim-appengine-backend
+  - UNIMPLEMENTED: mask is not supported yet
+  - UNIMPLEMENTED: target_id should always be 1 is not supported yet
+- polis-soc-reg-backend - ???

crates/googleapis/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "googleapis"
+edition = "2021"
+
+[dependencies]
+prost = "0.12"
+prost-types = "0.12"
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tonic = { workspace = true, features = ["gzip"] }
+
+[build-dependencies]
+tonic-build = "0.11"

build.rs → crates/googleapis/build.rs

@@ -3,8 +3,8 @@ fn main() {
         .build_client(false)
         .include_file("googleapis.rs")
         .compile(
-            &["googleapis/google/firestore/v1/firestore.proto"],
-            &["googleapis"],
+            &["include/google/firestore/v1/firestore.proto"],
+            &["include"],
         )
         .unwrap()
 }

crates/googleapis/src/lib.rs

@@ -0,0 +1,9 @@
+pub use googleapis::*;
+pub use prost_types::Timestamp;
+pub use timestamp_utils::*;
+
+mod googleapis {
+    tonic::include_proto!("googleapis");
+}
+mod timestamp_utils;
+mod value_ext;

crates/googleapis/src/timestamp_utils.rs

@@ -0,0 +1,54 @@
+use std::{sync::Mutex, time::SystemTime};
+
+use prost_types::Timestamp;
+use tonic::Status;
+
+use crate::google::firestore::v1::target;
+
+const NANOS_PER_SECOND: i128 = 1_000_000_000;
+
+pub fn timestamp_nanos(ts: &Timestamp) -> i128 {
+    ts.seconds as i128 * NANOS_PER_SECOND + ts.nanos as i128
+}
+
+pub fn timestamp_from_nanos(nanos: i128) -> Timestamp {
+    Timestamp {
+        seconds: (nanos / NANOS_PER_SECOND) as _,
+        nanos:   (nanos % NANOS_PER_SECOND) as _,
+    }
+}
+
+/// Returns the current time as a `Timestamp` with the added guarantee that all calls will return a
+/// strictly higher timestamp than all calls before that (for the execution of the program).
+pub fn timestamp() -> Timestamp {
+    static LAST: Mutex<Timestamp> = Mutex::new(Timestamp {
+        seconds: 0,
+        nanos:   0,
+    });
+    let mut last = LAST.lock().unwrap();
+    let mut timestamp = SystemTime::now().into();
+    if timestamp_nanos(&timestamp) <= timestamp_nanos(&last) {
+        timestamp = Timestamp {
+            seconds: last.seconds,
+            nanos:   last.nanos + 1,
+        }
+    }
+    last.clone_from(&timestamp);
+    timestamp
+}
+
+impl TryFrom<target::ResumeType> for Timestamp {
+    type Error = Status;
+
+    fn try_from(value: target::ResumeType) -> Result<Self, Self::Error> {
+        match value {
+            target::ResumeType::ResumeToken(token) => {
+                let token = token
+                    .try_into()
+                    .map_err(|_| Status::invalid_argument("invalid resume token"))?;
+                Ok(timestamp_from_nanos(i128::from_ne_bytes(token)))
+            }
+            target::ResumeType::ReadTime(time) => Ok(time),
+        }
+    }
+}

src/database/value.rs → crates/googleapis/src/value_ext.rs

@@ -8,7 +8,7 @@ use crate::{
         firestore::v1::{value::ValueType, ArrayValue, MapValue},
         r#type::LatLng,
     },
-    utils::timestamp_nanos,
+    timestamp_nanos,
 };
 
 impl Value {

justfile

@@ -0,0 +1,11 @@
+set quiet := true
+
+[private]
+default:
+    just --choose --justfile {{ justfile() }}
+
+alias t:= test
+
+test:
+    cargo nextest run
+    cargo test --doc

src/database.rs

@@ -4,8 +4,14 @@ use std::{
     sync::{Arc, Weak},
 };
 
+use googleapis::{
+    google::firestore::v1::{
+        document_transform::field_transform::{ServerValue, TransformType},
+        *,
+    },
+    timestamp, Timestamp,
+};
 use itertools::Itertools;
-use prost_types::Timestamp;
 use string_cache::DefaultAtom;
 use tokio::sync::{
     broadcast::{self, Receiver},
@@ -25,12 +31,7 @@ use self::{
     transaction::{RunningTransactions, Transaction, TransactionId},
 };
 use crate::{
-    googleapis::google::firestore::v1::{
-        document_transform::field_transform::{ServerValue, TransformType},
-        *,
-    },
-    unimplemented, unimplemented_collection, unimplemented_option,
-    utils::{timestamp, RwLockHashMapExt},
+    unimplemented, unimplemented_collection, unimplemented_option, utils::RwLockHashMapExt,
 };
 
 mod collection;
@@ -40,7 +41,6 @@ mod field_path;
 mod listener;
 mod query;
 mod transaction;
-mod value;
 
 const MAX_EVENT_BACKLOG: usize = 1024;
 

src/database/document.rs

@@ -7,7 +7,10 @@ use std::{
 };
 
 use futures::Future;
-use prost_types::Timestamp;
+use googleapis::{
+    google::firestore::v1::{precondition, Document, Value},
+    timestamp_nanos, Timestamp,
+};
 use string_cache::DefaultAtom;
 use tokio::{
     sync::{
@@ -20,10 +23,6 @@ use tonic::{Code, Result, Status};
 use tracing::{instrument, trace, Level};
 
 use super::ReadConsistency;
-use crate::{
-    googleapis::google::firestore::v1::{precondition, Document, Value},
-    utils::timestamp_nanos,
-};
 
 const WAIT_LOCK_TIMEOUT: Duration = Duration::from_secs(30);
 

src/database/event.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use prost_types::Timestamp;
+use googleapis::Timestamp;
 use string_cache::DefaultAtom;
 
 use super::document::DocumentVersion;

src/database/field_path.rs

@@ -1,9 +1,9 @@
 use std::{borrow::Cow, collections::HashMap, convert::Infallible, mem::take, ops::Deref};
 
+use googleapis::google::firestore::v1::*;
 use tonic::{Result, Status};
 
 use super::document::StoredDocumentVersion;
-use crate::googleapis::google::firestore::v1::*;
 
 /// The virtual field-name that represents the document-name.
 pub const DOC_NAME: &str = "__name__";

src/database/listener.rs

@@ -6,8 +6,17 @@ use std::{
     },
 };
 
+use googleapis::{
+    google::firestore::v1::{
+        listen_request,
+        listen_response::ResponseType,
+        target::{self, query_target},
+        target_change::TargetChangeType,
+        DocumentChange, DocumentDelete, ListenRequest, ListenResponse, Target, TargetChange,
+    },
+    timestamp, timestamp_nanos, Timestamp,
+};
 use itertools::Itertools;
-use prost_types::Timestamp;
 use string_cache::DefaultAtom;
 use tokio::sync::{broadcast::error::RecvError, mpsc};
 use tokio_stream::{wrappers::ReceiverStream, StreamExt};
@@ -18,17 +27,8 @@ use super::{
     document::DocumentVersion, event::DatabaseEvent, query::Query, target_change, Database,
 };
 use crate::{
-    database::ReadConsistency,
-    googleapis::google::firestore::v1::{
-        listen_request,
-        listen_response::ResponseType,
-        target::{self, query_target},
-        target_change::TargetChangeType,
-        DocumentChange, DocumentDelete, ListenRequest, ListenResponse, Target, TargetChange,
-    },
-    required_option, unimplemented, unimplemented_bool, unimplemented_collection,
-    unimplemented_option,
-    utils::{timestamp, timestamp_from_nanos, timestamp_nanos},
+    database::ReadConsistency, required_option, unimplemented, unimplemented_bool,
+    unimplemented_collection, unimplemented_option,
 };
 
 const TARGET_ID: i32 = 1;
@@ -481,19 +481,3 @@ fn show_response_type(rt: &ResponseType) -> &'static str {
         ResponseType::Filter(_) => "Filter",
     }
 }
-
-impl TryFrom<target::ResumeType> for Timestamp {
-    type Error = Status;
-
-    fn try_from(value: target::ResumeType) -> Result<Self, Self::Error> {
-        match value {
-            target::ResumeType::ResumeToken(token) => {
-                let token = token
-                    .try_into()
-                    .map_err(|_| Status::invalid_argument("invalid resume token"))?;
-                Ok(timestamp_from_nanos(i128::from_ne_bytes(token)))
-            }
-            target::ResumeType::ReadTime(time) => Ok(time),
-        }
-    }
-}

src/database/query.rs

@@ -1,5 +1,6 @@
 use std::{cmp, collections::HashMap, ops::Deref, sync::Arc};
 
+use googleapis::google::firestore::v1::{structured_query::CollectionSelector, *};
 use itertools::Itertools;
 use string_cache::DefaultAtom;
 use tonic::{Result, Status};
@@ -9,7 +10,6 @@ use super::{
     collection::Collection, document::StoredDocumentVersion, field_path::FieldReference, Database,
     ReadConsistency,
 };
-use crate::googleapis::google::firestore::v1::{structured_query::CollectionSelector, *};
 
 mod filter;
 

src/database/query/filter.rs

@@ -1,11 +1,11 @@
 use std::ops::Deref;
 
+use googleapis::google::firestore::v1::{structured_query, Value};
 use itertools::Itertools;
 use tonic::{Result, Status};
 
 use crate::{
-    database::{document::StoredDocumentVersion, field_path::FieldReference, value::Value},
-    googleapis::google::firestore::v1::structured_query,
+    database::{document::StoredDocumentVersion, field_path::FieldReference},
     unimplemented,
 };