Merge branch 'nightly' into preston/da_interface

Cargo.toml

@@ -37,6 +37,7 @@ members = [
     "module-system/module-implementations/module-template",
     "module-system/module-implementations/examples/sov-value-setter",
     "module-system/module-implementations/examples/sov-vec-setter",
+    "module-system/module-implementations/examples/sov-accessory-state",
     "module-system/module-implementations/integration-tests",
 ]
 exclude = [

full-node/db/sov-db/src/lib.rs

@@ -18,3 +18,8 @@ pub mod schema;
 /// Implements a wrapper around RocksDB meant for storing rollup state. This is primarily used
 /// as the backing store for the JMT.
 pub mod state_db;
+
+/// Implements a wrapper around RocksDB meant for storing state only accessible
+/// outside of the zkVM execution environment, as this data is not included in
+/// the JMT and does not contribute to proofs of execution.
+pub mod native_db;

full-node/db/sov-db/src/native_db.rs

@@ -0,0 +1,98 @@
+use std::path::Path;
+use std::sync::Arc;
+
+use sov_schema_db::{SchemaBatch, DB};
+
+use crate::rocks_db_config::gen_rocksdb_options;
+use crate::schema::tables::{ModuleAccessoryState, NATIVE_TABLES};
+use crate::schema::types::StateKey;
+
+/// A typed wrapper around RocksDB for storing native-only accessory state.
+/// Internally, this is roughly just an [`Arc<SchemaDB>`].
+#[derive(Clone)]
+pub struct NativeDB {
+    /// The underlying RocksDB instance, wrapped in an [`Arc`] for convenience
+    /// and [`DB`] for type safety.
+    db: Arc<DB>,
+}
+
+impl NativeDB {
+    const DB_PATH_SUFFIX: &str = "native";
+    const DB_NAME: &str = "native-db";
+
+    /// Opens a [`NativeDB`] (backed by RocksDB) at the specified path.
+    /// The returned instance will be at the path `{path}/native-db`.
+    pub fn with_path(path: impl AsRef<Path>) -> Result<Self, anyhow::Error> {
+        let path = path.as_ref().join(Self::DB_PATH_SUFFIX);
+        let inner = DB::open(
+            path,
+            Self::DB_NAME,
+            NATIVE_TABLES.iter().copied(),
+            &gen_rocksdb_options(&Default::default(), false),
+        )?;
+
+        Ok(Self {
+            db: Arc::new(inner),
+        })
+    }
+
+    /// Queries for a value in the [`NativeDB`], given a key.
+    pub fn get_value_option(&self, key: &StateKey) -> anyhow::Result<Option<Vec<u8>>> {
+        self.db
+            .get::<ModuleAccessoryState>(key)
+            .map(Option::flatten)
+    }
+
+    /// Sets a key-value pair in the [`NativeDB`].
+    pub fn set_value(&self, key: Vec<u8>, value: Option<Vec<u8>>) -> anyhow::Result<()> {
+        self.set_values(vec![(key, value)])
+    }
+
+    /// Sets a sequence of key-value pairs in the [`NativeDB`]. The write is atomic.
+    pub fn set_values(
+        &self,
+        key_value_pairs: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+    ) -> anyhow::Result<()> {
+        let batch = SchemaBatch::default();
+        for (key, value) in key_value_pairs {
+            batch.put::<ModuleAccessoryState>(&key, &value)?;
+        }
+        self.db.write_schemas(batch)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn get_after_set() {
+        let tmpdir = tempfile::tempdir().unwrap();
+        let db = NativeDB::with_path(tmpdir.path()).unwrap();
+
+        let key = b"foo".to_vec();
+        let value = b"bar".to_vec();
+        db.set_values(vec![(key.clone(), Some(value.clone()))])
+            .unwrap();
+        assert_eq!(db.get_value_option(&key).unwrap(), Some(value));
+    }
+
+    #[test]
+    fn get_after_delete() {
+        let tmpdir = tempfile::tempdir().unwrap();
+        let db = NativeDB::with_path(tmpdir.path()).unwrap();
+
+        let key = b"deleted".to_vec();
+        db.set_value(key.clone(), None).unwrap();
+        assert_eq!(db.get_value_option(&key).unwrap(), None);
+    }
+
+    #[test]
+    fn get_nonexistent() {
+        let tmpdir = tempfile::tempdir().unwrap();
+        let db = NativeDB::with_path(tmpdir.path()).unwrap();
+
+        let key = b"spam".to_vec();
+        assert_eq!(db.get_value_option(&key).unwrap(), None);
+    }
+}

full-node/db/sov-db/src/rocks_db_config.rs

@@ -42,6 +42,7 @@ pub fn gen_rocksdb_options(config: &RocksdbConfig, readonly: bool) -> Options {
     if !readonly {
         db_opts.create_if_missing(true);
         db_opts.create_missing_column_families(true);
+        db_opts.set_atomic_flush(true);
     }
 
     db_opts

full-node/db/sov-db/src/schema/tables.rs

@@ -1,5 +1,6 @@
 //! This module defines the following tables:
 //!
+//!
 //! Slot Tables:
 //! - `SlotNumber -> StoredSlot`
 //! - `SlotNumber -> Vec<BatchNumber>`
@@ -20,6 +21,9 @@
 //! - `KeyHash -> Key`
 //! - `(Key, Version) -> JmtValue`
 //! - `NodeKey -> Node`
+//!
+//! Module Accessory State Table:
+//! - `(ModuleAddress, Key) -> Value`
 
 use borsh::{maybestd, BorshDeserialize, BorshSerialize};
 use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
@@ -30,8 +34,8 @@ use sov_schema_db::schema::{KeyDecoder, KeyEncoder, ValueCodec};
 use sov_schema_db::{CodecError, SeekKeyEncoder};
 
 use super::types::{
-    BatchNumber, DbHash, EventNumber, JmtValue, SlotNumber, StateKey, StoredBatch, StoredSlot,
-    StoredTransaction, TxNumber,
+    AccessoryKey, AccessoryStateValue, BatchNumber, DbHash, EventNumber, JmtValue, SlotNumber,
+    StateKey, StoredBatch, StoredSlot, StoredTransaction, TxNumber,
 };
 
 /// A list of all tables used by the StateDB. These tables store rollup state - meaning
@@ -55,6 +59,11 @@ pub const LEDGER_TABLES: &[&str] = &[
     EventByNumber::table_name(),
 ];
 
+/// A list of all tables used by the NativeDB. These tables store
+/// "accessory" state only accessible from a native execution context, to be
+/// used for JSON-RPC and other tooling.
+pub const NATIVE_TABLES: &[&str] = &[ModuleAccessoryState::table_name()];
+
 /// Macro to define a table that implements [`sov_schema_db::Schema`].
 /// KeyCodec<Schema> and ValueCodec<Schema> must be implemented separately.
 ///
@@ -210,6 +219,11 @@ define_table_with_default_codec!(
     (SlotByHash) DbHash => SlotNumber
 );
 
+define_table_with_default_codec!(
+    /// Non-JMT state stored by a module for JSON-RPC use.
+    (ModuleAccessoryState) AccessoryKey => AccessoryStateValue
+);
+
 define_table_with_seek_key_codec!(
     /// The primary source for batch data
     (BatchByNumber) BatchNumber => StoredBatch

full-node/db/sov-db/src/schema/types.rs

@@ -37,6 +37,15 @@ impl AsRef<[u8]> for DbBytes {
     }
 }
 
+/// The "key" half of a key/value pair from accessory state.
+///
+/// See [`NativeDB`](crate::native_db::NativeDB) for more information.
+pub type AccessoryKey = Vec<u8>;
+/// The "value" half of a key/value pair from accessory state.
+///
+/// See [`NativeDB`](crate::native_db::NativeDB) for more information.
+pub type AccessoryStateValue = Option<Vec<u8>>;
+
 /// A hash stored in the database
 pub type DbHash = [u8; 32];
 /// The "value" half of a key/value pair from the JMT

module-system/module-implementations/examples/sov-accessory-state/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "sov-accessory-state"
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+readme = "README.md"
+publish = false
+resolver = "2"
+
+[dependencies]
+jsonrpsee = { workspace = true, features = ["macros", "client-core", "server"], optional = true }
+sov-modules-api = { path = "../../../sov-modules-api" }
+sov-state = { path = "../../../sov-state" }
+serde = { workspace = true, optional = true }
+borsh = { workspace = true, features = ["rc"] }
+
+[dev-dependencies]
+tempfile = { workspace = true }
+
+[features]
+default = []
+native = ["serde", "sov-modules-api/native", "dep:jsonrpsee"]

module-system/module-implementations/examples/sov-accessory-state/README.md

@@ -0,0 +1,5 @@
+# The `sov-accessory-state` module
+
+This module has no useful functionality, but it illustrates the ability to store data outside of the JMT, called "accessory state".
+
+Accessory state does not contribute to the state root hash and can be written during zkVM execution, but reads are only allowed in a native execution context. Accessory state data can be used for tooling, serving JSON-RPC data, debugging, and any other purpose that is not core to the core functioning of the module without incurring in major performance penalties.

module-system/module-implementations/examples/sov-accessory-state/src/lib.rs

@@ -0,0 +1,68 @@
+#![deny(missing_docs)]
+#![doc = include_str!("../README.md")]
+
+#[cfg(feature = "native")]
+pub mod query;
+
+use sov_modules_api::{CallResponse, Context, Error, Module, ModuleInfo};
+use sov_state::{AccessoryStateValue, StateValue, WorkingSet};
+
+/// [`AccessorySetter`] is a module that stores data both *inside* the JMT and
+/// *outside* the JMT.
+///
+/// Data stored inside the JMT contributes to the state root hash, and is always
+/// accessible. This costs significant compute, and should be avoided for all
+/// data that is not necessary to the core functioning of the rollup. Other data
+/// that facilitates serving queries over JSON-RPC or only accessed by tooling
+/// doesn't need to be verifiable, and can thus be stored outside the JMT much
+/// more cheaply. Since accessory data is not included in the state root hash,
+/// it is not accessible inside the zkVM and can only be accessed with
+/// `#[cfg(feature = "native")]`.
+#[derive(ModuleInfo)]
+pub struct AccessorySetter<C: sov_modules_api::Context> {
+    /// The address of the module.
+    #[address]
+    pub address: C::Address,
+    /// Some arbitrary value stored in the JMT to demonstrate the difference
+    /// between the JMT and accessory state.
+    #[state]
+    pub state_value: StateValue<String>,
+    /// A non-JMT value stored in the accessory state.
+    #[state]
+    pub accessory_value: AccessoryStateValue<String>,
+}
+
+/// The [`Module::CallMessage`] for [`AccessorySetter`].
+#[derive(borsh::BorshDeserialize, borsh::BorshSerialize, Debug, PartialEq)]
+pub enum CallMessage {
+    /// Sets the value of [`AccessorySetter::state_value`].
+    SetValue(String),
+    /// Stores some arbitrary value in the accessory state.
+    SetValueAccessory(String),
+}
+
+impl<C: Context> Module for AccessorySetter<C> {
+    type Context = C;
+
+    type Config = ();
+
+    type CallMessage = CallMessage;
+
+    fn call(
+        &self,
+        msg: Self::CallMessage,
+        _context: &Self::Context,
+        working_set: &mut WorkingSet<C::Storage>,
+    ) -> Result<sov_modules_api::CallResponse, Error> {
+        match msg {
+            CallMessage::SetValueAccessory(new_value) => {
+                self.accessory_value
+                    .set(&new_value, &mut working_set.accessory_state());
+            }
+            CallMessage::SetValue(new_value) => {
+                self.state_value.set(&new_value, working_set);
+            }
+        };
+        Ok(CallResponse::default())
+    }
+}

module-system/module-implementations/examples/sov-accessory-state/src/query.rs

@@ -0,0 +1,29 @@
+//! JSON-RPC server implementation for the [`AccessorySetter`] module.
+
+use jsonrpsee::core::RpcResult;
+use sov_modules_api::macros::rpc_gen;
+use sov_state::WorkingSet;
+
+use super::AccessorySetter;
+
+/// Response type to the `accessorySetter_value` endpoint.
+#[derive(Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize, Clone)]
+pub struct ValueResponse {
+    /// The value stored in the accessory state.
+    pub value: Option<String>,
+}
+
+#[rpc_gen(client, server, namespace = "accessorySetter")]
+impl<C: sov_modules_api::Context> AccessorySetter<C> {
+    /// Returns the latest value set in the accessory state via
+    /// [`CallMessage::SetValueAccessory`](crate::CallMessage::SetValueAccessory).
+    #[rpc_method(name = "value")]
+    pub fn query_value(
+        &self,
+        working_set: &mut WorkingSet<C::Storage>,
+    ) -> RpcResult<ValueResponse> {
+        Ok(ValueResponse {
+            value: self.accessory_value.get(&mut working_set.accessory_state()),
+        })
+    }
+}