perf(507): Add disk and memory usage metrics

Closes #507.

crates/core/src/database_instance_context.rs

@@ -6,6 +6,7 @@ use crate::db::ostorage::sled_object_db::SledObjectDB;
 use crate::db::ostorage::ObjectDB;
 use crate::db::relational_db::RelationalDB;
 use crate::db::{Config, FsyncPolicy, Storage};
+use crate::error::DBError;
 use crate::identity::Identity;
 use crate::messages::control_db::Database;
 use std::path::{Path, PathBuf};
@@ -93,4 +94,19 @@ impl DatabaseInstanceContext {
     pub(crate) fn make_default_ostorage(path: impl AsRef<Path>) -> Box<dyn ObjectDB + Send> {
         Box::new(SledObjectDB::open(path).unwrap())
     }
+
+    /// The number of bytes on disk occupied by the [MessageLog].
+    pub fn message_log_size_on_disk(&self) -> Result<u64, DBError> {
+        self.relational_db.commit_log().message_log_size_on_disk()
+    }
+
+    /// The number of bytes on disk occupied by the [ObjectDB].
+    pub fn object_db_size_on_disk(&self) -> Result<u64, DBError> {
+        self.relational_db.commit_log().object_db_size_on_disk()
+    }
+
+    /// The size of the log file.
+    pub fn log_file_size(&self) -> Result<u64, DBError> {
+        self.logger.size()
+    }
 }

crates/core/src/database_instance_context_controller.rs

@@ -1,6 +1,7 @@
 use std::sync::Arc;
 use std::{collections::HashMap, sync::Mutex};
 
+use crate::db::db_metrics::DB_METRICS;
 use crate::host::scheduler::Scheduler;
 
 use super::database_instance_context::DatabaseInstanceContext;
@@ -33,4 +34,32 @@ impl DatabaseInstanceContextController {
         let mut contexts = self.contexts.lock().unwrap();
         contexts.remove(&database_instance_id)
     }
+
+    #[tracing::instrument(skip_all)]
+    pub fn update_metrics(&self) {
+        let contexts = self.contexts.lock().unwrap();
+        for (db, _) in contexts.values() {
+            // Use the previous gauge value if there is an issue getting the file size.
+            if let Ok(num_bytes) = db.message_log_size_on_disk() {
+                DB_METRICS
+                    .message_log_disk_usage
+                    .with_label_values(&db.address)
+                    .set(num_bytes as i64);
+            }
+            // Use the previous gauge value if there is an issue getting the file size.
+            if let Ok(num_bytes) = db.object_db_size_on_disk() {
+                DB_METRICS
+                    .object_db_disk_usage
+                    .with_label_values(&db.address)
+                    .set(num_bytes as i64);
+            }
+            // Use the previous gauge value if there is an issue getting the file size.
+            if let Ok(num_bytes) = db.log_file_size() {
+                DB_METRICS
+                    .module_logger_disk_usage
+                    .with_label_values(&db.address)
+                    .set(num_bytes as i64);
+            }
+        }
+    }
 }

crates/core/src/db/commit_log.rs

@@ -188,6 +188,22 @@ pub struct CommitLogView {
 }
 
 impl CommitLogView {
+    /// The number of bytes on disk occupied by the [MessageLog].
+    pub fn message_log_size_on_disk(&self) -> Result<u64, DBError> {
+        if let Some(ref mlog) = self.mlog {
+            let guard = mlog.lock().unwrap();
+            Ok(guard.size())
+        } else {
+            Ok(0)
+        }
+    }
+
+    /// The number of bytes on disk occupied by the [ObjectDB].
+    pub fn object_db_size_on_disk(&self) -> Result<u64, DBError> {
+        let guard = self.odb.lock().unwrap();
+        guard.size_on_disk()
+    }
+
     /// Obtain an iterator over a snapshot of the raw message log segments.
     ///
     /// See also: [`MessageLog::segments`]

crates/core/src/db/db_metrics/mod.rs

@@ -110,6 +110,21 @@ metrics_group!(
         #[help = "The total duration of a spacetime wasm abi call (in seconds); includes row serialization and copying into wasm memory"]
         #[labels(txn_type: TransactionType, db: Address, reducer: str, call: AbiCall)]
         pub wasm_abi_call_duration_sec: HistogramVec,
+
+        #[name = spacetime_module_log_size_bytes]
+        #[help = "For a given module, the size of its log file (in bytes)"]
+        #[labels(db: Address)]
+        pub module_logger_disk_usage: IntGaugeVec,
+
+        #[name = spacetime_message_log_size_bytes]
+        #[help = "For a given database, the number of bytes occupied by its message log"]
+        #[labels(db: Address)]
+        pub message_log_disk_usage: IntGaugeVec,
+
+        #[name = spacetime_message_log_size_bytes]
+        #[help = "For a given database, the number of bytes occupied by large object storage"]
+        #[labels(db: Address)]
+        pub object_db_disk_usage: IntGaugeVec,
     }
 );
 

crates/core/src/db/ostorage/hashmap_object_db.rs

@@ -186,6 +186,10 @@ impl ObjectDB for HashMapObjectDB {
         }
         Ok(())
     }
+
+    fn size_on_disk(&self) -> Result<u64, DBError> {
+        Ok(self.total_mem_size_bytes())
+    }
 }
 
 fn hex_prefixes() -> Vec<String> {

crates/core/src/db/ostorage/memory_object_db.rs

@@ -31,4 +31,8 @@ impl ObjectDB for MemoryObjectDB {
     fn sync_all(&mut self) -> Result<(), crate::error::DBError> {
         Ok(())
     }
+
+    fn size_on_disk(&self) -> Result<u64, crate::error::DBError> {
+        Ok(0)
+    }
 }

crates/core/src/db/ostorage/mod.rs

@@ -20,4 +20,5 @@ pub trait ObjectDB {
     fn get(&self, hash: Hash) -> Option<bytes::Bytes>;
     fn flush(&mut self) -> Result<(), DBError>;
     fn sync_all(&mut self) -> Result<(), DBError>;
+    fn size_on_disk(&self) -> Result<u64, DBError>;
 }

crates/core/src/db/ostorage/rocks_object_db.rs

@@ -66,6 +66,11 @@ impl ObjectDB for RocksDBObjectDB {
     fn sync_all(&mut self) -> Result<(), DBError> {
         self.flush()
     }
+
+    fn size_on_disk(&self) -> Result<u64, DBError> {
+        // TODO: Compute the size of the rocksdb instance
+        Ok(0)
+    }
 }
 
 #[cfg(test)]

crates/core/src/db/ostorage/sled_object_db.rs

@@ -17,7 +17,6 @@ impl SledObjectDB {
             .flush_every_ms(Some(50))
             .mode(HighThroughput);
         let db = config.open()?;
-
         Ok(Self { db })
     }
 }
@@ -48,6 +47,10 @@ impl ObjectDB for SledObjectDB {
     fn sync_all(&mut self) -> Result<(), DBError> {
         self.flush()
     }
+
+    fn size_on_disk(&self) -> Result<u64, DBError> {
+        Ok(self.db.size_on_disk()?)
+    }
 }
 
 #[cfg(test)]

crates/standalone/src/lib.rs

@@ -163,6 +163,8 @@ fn get_key_path(env: &str) -> Option<PathBuf> {
 #[async_trait]
 impl spacetimedb_client_api::NodeDelegate for StandaloneEnv {
     fn gather_metrics(&self) -> Vec<prometheus::proto::MetricFamily> {
+        // Note, we update certain metrics such as disk usage on demand.
+        self.db_inst_ctx_controller.update_metrics();
         self.metrics_registry.gather()
     }